diff --git a/.vscode/settings.json b/.vscode/settings.json index 0744e9d78..2af9246fe 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -103,6 +103,12 @@ "charconv": "cpp", "source_location": "cpp", "strstream": "cpp", - "typeindex": "cpp" + "typeindex": "cpp", + "*.tcc": "cpp", + "memory_resource": "cpp", + "numbers": "cpp", + "semaphore": "cpp", + "stop_token": "cpp", + "cfenv": "cpp" } } \ No newline at end of file diff --git a/doc/basics.md b/doc/basics.md index 73a4249fe..954406e4c 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -1910,8 +1910,10 @@ An `ondemand::number` instance may contain an integer value or a floating-point Thus it is a dynamically typed number. Before accessing the value, you must determine the detected type: * `number.get_number_type()` has value `number_type::signed_integer` if we have a integer in [-9223372036854775808,9223372036854775808). You can recover the value by the `get_int64()` method applied on the `ondemand::number` instance. When `number.get_number_type()` has value `number_type::signed_integer`, you also have that `number.is_int64()` is true. Calling `get_int64()` on the `ondemand::number` instance when `number.get_number_type()` is not `number_type::signed_integer` is unsafe. You may replace `get_int64()` by a cast to a `int64_t` value. -* `number.get_number_type()` has value `number_type::unsigned_integer` if we have a integer in [9223372036854775808,18446744073709551616). You can recover the value by the `get_uint64()` method applied on the `ondemand::number` instance. When `number.get_number_type()` has value `number_type::unsigned_integer`, you also have that `number.is_uint64()` is true. Calling `get_uint64()` on the `ondemand::number` instance when `number.get_number_type()` is not `number_type::unsigned_integer` is unsafe. You may replace `get_uint64()` by a cast to a `uint64_t` value. +* `number.get_number_type()` has value `number_type::unsigned_integer` if we have a integer in `[9223372036854775808,18446744073709551616)`. You can recover the value by the `get_uint64()` method applied on the `ondemand::number` instance. When `number.get_number_type()` has value `number_type::unsigned_integer`, you also have that `number.is_uint64()` is true. Calling `get_uint64()` on the `ondemand::number` instance when `number.get_number_type()` is not `number_type::unsigned_integer` is unsafe. You may replace `get_uint64()` by a cast to a `uint64_t` value. * `number.get_number_type()` has value `number_type::floating_point_number` if we have and we have a floating-point (binary64) number. You can recover the value by the `get_double()` method applied on the `ondemand::number` instance. When `number.get_number_type()` has value `number_type::floating_point_number`, you also have that `number.is_double()` is true. Calling `get_double()` on the `ondemand::number` instance when `number.get_number_type()` is not `number_type::floating_point_number` is unsafe. You may replace `get_double()` by a cast to a `double` value. +* When the value is an integer outside of the valid ranges for a 64-bit integers, e.g., when it is smaller than -9223372036854775808 or larger than 18446744073709551615, then `number.get_number_type()` has value `number_type::big_integer`. If you try to parse +such a number of `get_number()`, you get the error `BIGINT_ERROR`. You can access the underlying string of digits with the function `raw_json_token()` which returns an `std::string_view` instance starting at the beginning of the digit. You can also call `get_double()` to get a floating-point approximation. You must check the type before accessing the value: it is an error to call `get_int64()` when `number.get_number_type()` is not `number_type::signed_integer` and when `number.is_int64()` is false. You are responsible for this check as the user of the library. @@ -1944,6 +1946,9 @@ Consider the following example: std::cout << "float: " << double(num) << " "; std::cout << "float: " << num.get_double() << std::endl; break; + case ondemand::number_type::big_integer: + std::cout << "big-integer: " << val.raw_json_token() << std::endl; + break; } } ``` @@ -1959,6 +1964,55 @@ It will output: 9999999999999999999 negative: 0 is_integer: 1 large 64-bit integer: 9999999999999999999 large 64-bit integer: 9999999999999999999 ``` +In the following example, we have an array of integers that are outside the valid range of 64-bit signed or +unsigned integers. Calling `get_number_type()` on the values returns `ondemand::number_type::big_integer`. +You can try to represent these big integers as 64-bit floating-point numbers, though you typically lose +precision in the process (as illustrated in the example). + +```C++ + ondemand::parser parser; + padded_string docdata = R"([-9223372036854775809, 18446744073709551617, 99999999999999999999999 ])"_padded; + double dexpected[] = {-9223372036854775808.0, 18446744073709551616.0, 1e23}; + ondemand::document doc = parser.iterate(docdata); + ondemand::array arr = doc.get_array(); + for(ondemand::value val : arr) { + if(val.get_number_type() == ondemand::number_type::big_integer) { + std::cout << val.get_double() << std::endl; + // might print -9.22337e+18, 1.84467e+19, 1e+23 + } + } +``` +This program might print: +``` +-9.22337e+18 +1.84467e+19 +1e+23 +``` + +You may get access to the underlying string representing the big integer with +`raw_json_token()` and you may parse the resulting number strings using your own parser. + +```c++ + ondemand::parser parser; + padded_string docdata = R"([-9223372036854775809, 18446744073709551617, 99999999999999999999999 ])"_padded; + ondemand::document doc = parser.iterate(docdata); + ondemand::array arr = doc.get_array(); + for(ondemand::value val : arr) { + // val.get_number_type() == ondemand::number_type::big_integer + if(val.get_number_type() == ondemand::number_type::big_integer) { + std::string_view token = val.raw_json_token(); + // token = "-9223372036854775809", "18446744073709551617", "99999999999999999999999 " + std::cout << "'" << token << "'" << std::endl; + } + } +``` +This code prints the following: +``` +'-9223372036854775809' +'18446744073709551617' +'99999999999999999999999 ' +``` + Raw Strings From Keys ----------- diff --git a/include/simdjson/generic/numberparsing.h b/include/simdjson/generic/numberparsing.h index ef072325f..facc2acc6 100644 --- a/include/simdjson/generic/numberparsing.h +++ b/include/simdjson/generic/numberparsing.h @@ -357,6 +357,10 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) { return true; } +simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; +} + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer @@ -436,6 +440,23 @@ simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const s return SUCCESS; } +simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { + const uint8_t *const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t *p = src + uint8_t(negative); + if(p == srcend) { return false; } + if(*p == '0') { + ++p; + if(p == srcend) { return true; } + if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while(p != srcend && is_digit(*p)) { ++p; } + if(p == srcend) { return true; } + if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; +} + simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. @@ -1085,9 +1106,17 @@ simdjson_unused simdjson_inline simdjson_result get_number_type(con // If the number is negative and valid, it must be a signed integer. if(negative) { if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } return number_type::signed_integer; } + // Let us check if we have a big integer (>=2**64). + static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). // We want values larger or equal to 9223372036854775808 to be unsigned // integers, and the other values to be signed integers. if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { diff --git a/include/simdjson/generic/ondemand/document.h b/include/simdjson/generic/ondemand/document.h index 5a2fb4ea8..ee75f35ad 100644 --- a/include/simdjson/generic/ondemand/document.h +++ b/include/simdjson/generic/ondemand/document.h @@ -467,11 +467,11 @@ public: * get_number().get_number_type(). * * get_number_type() is number_type::unsigned_integer if we have - * an integer greater or equal to 9223372036854775808 + * an integer greater or equal to 9223372036854775808 and no larger than 18446744073709551615. * get_number_type() is number_type::signed_integer if we have an - * integer that is less than 9223372036854775808 - * get_number_type() is number_type::big_integer if we have and integer larger - * than those ranges above + * integer that is less than 9223372036854775808 and greater or equal to -9223372036854775808. + * get_number_type() is number_type::big_integer if we have an integer outside + * of those ranges (either larger than 18446744073709551615 or smaller than -9223372036854775808). * Otherwise, get_number_type() has value number_type::floating_point_number * * This function requires processing the number string, but it is expected diff --git a/include/simdjson/generic/ondemand/object.h b/include/simdjson/generic/ondemand/object.h index b21c10339..58852df83 100644 --- a/include/simdjson/generic/ondemand/object.h +++ b/include/simdjson/generic/ondemand/object.h @@ -77,7 +77,7 @@ public: * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field was not there when they are not in order). * * If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful * that only one field is returned. diff --git a/include/simdjson/generic/ondemand/value_iterator-inl.h b/include/simdjson/generic/ondemand/value_iterator-inl.h index 00d5e95e1..a8ed2969a 100644 --- a/include/simdjson/generic/ondemand/value_iterator-inl.h +++ b/include/simdjson/generic/ondemand/value_iterator-inl.h @@ -137,7 +137,7 @@ simdjson_warn_unused simdjson_inline simdjson_result value_iterator::find_ } else if (!is_open()) { #if SIMDJSON_DEVELOPMENT_CHECKS // If we're past the end of the object, we're being iterated out of order. - // Note: this isn't perfect detection. It's possible the user is inside some other object; if so, + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, // this object iterator will blithely scan that object for fields. if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } #endif @@ -249,7 +249,7 @@ simdjson_warn_unused simdjson_inline simdjson_result value_iterator::find_ #if SIMDJSON_DEVELOPMENT_CHECKS // If we're past the end of the object, we're being iterated out of order. - // Note: this isn't perfect detection. It's possible the user is inside some other object; if so, + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, // this object iterator will blithely scan that object for fields. if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } #endif @@ -615,7 +615,12 @@ simdjson_inline simdjson_result value_iter uint8_t tmpbuf[1074+8+1+1]; tmpbuf[1074+8+1] = '\0'; // make sure that buffer is always null terminated. if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074+8+1)) { - logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + if(numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return number_type::big_integer; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); return NUMBER_ERROR; } auto answer = numberparsing::get_number_type(tmpbuf); @@ -632,7 +637,12 @@ simdjson_inline simdjson_result value_iterator::get_root_number(bool che uint8_t tmpbuf[1074+8+1+1]; tmpbuf[1074+8+1] = '\0'; // make sure that buffer is always null terminated. if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074+8+1)) { - logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + if(numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return BIGINT_ERROR; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); return NUMBER_ERROR; } number num; diff --git a/tests/ondemand/ondemand_number_tests.cpp b/tests/ondemand/ondemand_number_tests.cpp index 051e89b43..6ed32cf04 100644 --- a/tests/ondemand/ondemand_number_tests.cpp +++ b/tests/ondemand/ondemand_number_tests.cpp @@ -464,8 +464,50 @@ namespace number_tests { TEST_SUCCEED(); } + bool big_int_not_zero() { + TEST_START(); + ondemand::parser parser; + ondemand::document doc; + // This is not a big integer, it is a mistake + padded_string docdata = R"(09500000000000000000000000000000000000)"_padded; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + ASSERT_ERROR(doc.get_number(), NUMBER_ERROR); + TEST_SUCCEED(); + } + + bool negative_big_int() { + TEST_START(); + ondemand::parser parser; + ondemand::document doc; + // This is not a big integer, it is a mistake + padded_string docdata = R"(-18446744073709551616)"_padded; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + ASSERT_ERROR(doc.get_number(), BIGINT_ERROR); + std::string_view my_big; + ASSERT_SUCCESS(doc.raw_json_token().get(my_big)); + ASSERT_EQUAL(my_big, "-18446744073709551616"); + TEST_SUCCEED(); + } + + bool gigantic_big_int() { + TEST_START(); + ondemand::parser parser; + ondemand::document doc; + std::string number(2000, '1'); + // This is not a big integer, it is a mistake + ASSERT_SUCCESS(parser.iterate(number).get(doc)); + ASSERT_ERROR(doc.get_number(), BIGINT_ERROR); + std::string_view my_big; + ASSERT_SUCCESS(doc.raw_json_token().get(my_big)); + ASSERT_EQUAL(my_big, number); + TEST_SUCCEED(); + } + bool run() { - return issue2099() && + return gigantic_big_int() && + big_int_not_zero() && + negative_big_int() && + issue2099() && issue2093() && issue2045() && issue2017() && diff --git a/tests/ondemand/ondemand_readme_examples.cpp b/tests/ondemand/ondemand_readme_examples.cpp index 9b7fbe1b3..9c24441c2 100644 --- a/tests/ondemand/ondemand_readme_examples.cpp +++ b/tests/ondemand/ondemand_readme_examples.cpp @@ -3,6 +3,9 @@ #if __cpp_lib_optional >= 201606L #include #endif +#if SIMDJSON_CPLUSPLUS17 +#include +#endif using namespace std; using namespace simdjson; using error_code=simdjson::error_code; @@ -382,6 +385,67 @@ bool examplecrt_realloc() { TEST_SUCCEED(); } +#if SIMDJSON_CPLUSPLUS17 +bool big_int_array() { + TEST_START(); + ondemand::parser parser; + padded_string docdata = R"([-9223372036854775809, 18446744073709551617, 99999999999999999999999 ])"_padded; + std::string expected[] = {"-9223372036854775809", "18446744073709551617", "99999999999999999999999 "}; + ondemand::document doc = parser.iterate(docdata); + ondemand::array arr = doc.get_array(); + size_t i = 0; + for(ondemand::value val : arr) { + if(i > 3) { + std::cerr << "unexpected number of elements" << std::endl; + return false; + } + if(val.get_number_type() != ondemand::number_type::big_integer) { + std::cerr << "unexpected number type" << std::endl; + std::cout << val.get_number_type() << std::endl; + std::cout << val.raw_json_token() << std::endl; + return false; + } + std::string_view token = val.raw_json_token(); + std::string_view expected_token = expected[i]; + if(token != expected_token) { + std::cerr << "unexpected token: " << token << " expected: " << expected_token << std::endl; + return false; + } + i++; + } + if(i != 3) { + std::cerr << "unexpected number of elements" << std::endl; + return false; + } + TEST_SUCCEED(); +} +#endif + +bool big_int_array_as_double() { + TEST_START(); + ondemand::parser parser; + padded_string docdata = R"([-9223372036854775809, 18446744073709551617, 99999999999999999999999 ])"_padded; + double dexpected[] = {-9223372036854775808.0, 18446744073709551616.0, 1e23}; + ondemand::document doc = parser.iterate(docdata); + ondemand::array arr = doc.get_array(); + size_t i = 0; + for(ondemand::value val : arr) { + if(i > 3) { + std::cerr << "unexpected number of elements" << std::endl; + return false; + } + if((val.get_number_type() != ondemand::number_type::big_integer) || (dexpected[i] != val.get_double())) { + return false; + } + i++; + } + if(i != 3) { + std::cerr << "unexpected number of elements" << std::endl; + return false; + } + TEST_SUCCEED(); +} + bool number_tests() { TEST_START(); ondemand::parser parser; @@ -1782,6 +1846,10 @@ bool value_raw_json_object() { bool run() { return true #if SIMDJSON_EXCEPTIONS +#if SIMDJSON_CPLUSPLUS17 + && big_int_array() +#endif // SIMDJSON_CPLUSPLUS17 + && big_int_array_as_double() && key_raw_json_token() && to_optional() && value_raw_json_array() && value_raw_json_object() @@ -1789,7 +1857,7 @@ bool run() { && at_end() && example1956() && example1958() && allow_comma_separated_example() -// && basics_1() // Fails because twitter.json isn't in current directory. Compile test only. +// && basics_1() // Fails because twitter.json is not in current directory. Compile test only. && basics_treewalk() && basics_treewalk_breakline() && json_value_with_array_count()