mirror of
https://github.com/simdjson/simdjson
synced 2026-06-08 17:27:07 +00:00
Compare commits
14 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e1c6a778f8 | |||
| d413dd5b0e | |||
| a33fdc2770 | |||
| 87f5f7a250 | |||
| 9923d27994 | |||
| 876be23545 | |||
| 2028bdaeac | |||
| 1ac3a00ca9 | |||
| 6d7cc2c7c1 | |||
| 61a52287fc | |||
| 37a9aaad48 | |||
| 78325888ea | |||
| a90cb5d900 | |||
| 34a01ec3af |
Vendored
+7
-1
@@ -103,6 +103,12 @@
|
||||
"charconv": "cpp",
|
||||
"source_location": "cpp",
|
||||
"strstream": "cpp",
|
||||
"typeindex": "cpp"
|
||||
"typeindex": "cpp",
|
||||
"*.tcc": "cpp",
|
||||
"memory_resource": "cpp",
|
||||
"numbers": "cpp",
|
||||
"semaphore": "cpp",
|
||||
"stop_token": "cpp",
|
||||
"cfenv": "cpp"
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
|
||||
project(
|
||||
simdjson
|
||||
# The version number is modified by tools/release.py
|
||||
VERSION 3.7.0
|
||||
VERSION 3.8.0
|
||||
DESCRIPTION "Parsing gigabytes of JSON per second"
|
||||
HOMEPAGE_URL "https://simdjson.org/"
|
||||
LANGUAGES CXX C
|
||||
@@ -20,8 +20,8 @@ string(
|
||||
# ---- Options, variables ----
|
||||
|
||||
# These version numbers are modified by tools/release.py
|
||||
set(SIMDJSON_LIB_VERSION "20.0.0" CACHE STRING "simdjson library version")
|
||||
set(SIMDJSON_LIB_SOVERSION "20" CACHE STRING "simdjson library soversion")
|
||||
set(SIMDJSON_LIB_VERSION "21.0.0" CACHE STRING "simdjson library version")
|
||||
set(SIMDJSON_LIB_SOVERSION "21" CACHE STRING "simdjson library soversion")
|
||||
|
||||
option(SIMDJSON_BUILD_STATIC_LIB "Build simdjson_static library along with simdjson" OFF)
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ PROJECT_NAME = simdjson
|
||||
# could be handy for archiving the generated documentation or if some version
|
||||
# control system is used.
|
||||
|
||||
PROJECT_NUMBER = "3.7.0"
|
||||
PROJECT_NUMBER = "3.8.0"
|
||||
|
||||
# Using the PROJECT_BRIEF tag one can provide an optional one line description
|
||||
# for a project that appears at the top of each page and should give viewer a
|
||||
|
||||
@@ -283,8 +283,6 @@ If your compiler does not default on C++11 support or better you may get failing
|
||||
|
||||
Note that the name of directory (`build`) is arbitrary, you can name it as you want (e.g., `buildgcc`) and you can have as many different such directories as you would like (one per configuration).
|
||||
|
||||
|
||||
|
||||
## Usage (CMake on 64-bit Windows using Visual Studio 2019 or better)
|
||||
|
||||
Recent versions of Visual Studio support CMake natively, [please refer to the Visual Studio documentation](https://learn.microsoft.com/en-us/cpp/build/cmake-projects-in-visual-studio?view=msvc-170).
|
||||
@@ -309,13 +307,11 @@ Though having Visual Studio installed is necessary, one can build simdjson using
|
||||
|
||||
Furthermore, if you have installed LLVM clang on Windows, for example as a component of Visual Studio 2019, you can configure and build simdjson using LLVM clang on Windows using cmake:
|
||||
|
||||
|
||||
- `mkdir build`
|
||||
- `cd build`
|
||||
- `cmake -T ClangCL ..`
|
||||
- `cmake --build . --config Release`
|
||||
|
||||
|
||||
## Various References
|
||||
|
||||
- [How to implement atoi using SIMD?](https://stackoverflow.com/questions/35127060/how-to-implement-atoi-using-simd)
|
||||
|
||||
@@ -169,6 +169,7 @@ We distinguish between "bindings" (which just wrap the C++ code) and a port to a
|
||||
- [JSON-Simd](https://github.com/rawleyfowler/JSON-simd): Raku bindings.
|
||||
- [JSON::SIMD](https://metacpan.org/pod/JSON::SIMD): Perl bindings; fully-featured JSON module that uses simdjson for decoding.
|
||||
- [gemmaJSON](https://github.com/sainttttt/gemmaJSON): Nim JSON parser based on simdjson bindings.
|
||||
- [simdjson-java](https://github.com/simdjson/simdjson-java): Java port.
|
||||
|
||||
About simdjson
|
||||
--------------
|
||||
|
||||
@@ -993,7 +993,6 @@ struct AppleEvents {
|
||||
// check permission
|
||||
int force_ctrs = 0;
|
||||
if (kpc_force_all_ctrs_get(&force_ctrs)) {
|
||||
printf("Permission denied, xnu/kpc requires root privileges.\n");
|
||||
return (worked = false);
|
||||
}
|
||||
int ret;
|
||||
|
||||
Vendored
+4
-4
@@ -22,7 +22,7 @@ endif()
|
||||
# The bulk of our benchmarking and testing data has been moved simdjson/simdjson-data
|
||||
import_dependency(simdjson-data simdjson/simdjson-data a5b13babe65c1bba7186b41b43d4cbdc20a5c470)
|
||||
add_dependency(simdjson-data)
|
||||
|
||||
option(SIMDJSON_USE_BOOST_JSON "Try to include BOOST_JSON, this may break your binaries under some systems." OFF)
|
||||
# This prevents variables declared with set() from unnecessarily escaping and
|
||||
# should not be called more than once
|
||||
function(competition_scope_)
|
||||
@@ -36,8 +36,8 @@ function(competition_scope_)
|
||||
#endif
|
||||
|
||||
int main() {}
|
||||
]] USE_BOOST_JSON)
|
||||
if(USE_BOOST_JSON)
|
||||
]] SIMDJSON_FOUND_STRING_VIEW)
|
||||
if(SIMDJSON_FOUND_STRING_VIEW AND SIMDJSON_USE_BOOST_JSON)
|
||||
import_dependency(boostjson boostorg/json ee8d72d)
|
||||
add_library(boostjson STATIC "${boostjson_SOURCE_DIR}/src/src.cpp")
|
||||
target_compile_definitions(boostjson PUBLIC BOOST_JSON_STANDALONE)
|
||||
@@ -134,7 +134,7 @@ int main() {}
|
||||
add_library(competition-core INTERFACE)
|
||||
target_link_libraries(competition-core INTERFACE nlohmann_json rapidjson sajson cjson jsmn yyjson)
|
||||
|
||||
if(USE_BOOST_JSON)
|
||||
if(TARGET boostjson)
|
||||
target_compile_definitions(boostjson INTERFACE HAS_BOOST_JSON)
|
||||
target_link_libraries(competition-core INTERFACE boostjson)
|
||||
endif()
|
||||
|
||||
+55
-1
@@ -1910,8 +1910,10 @@ An `ondemand::number` instance may contain an integer value or a floating-point
|
||||
Thus it is a dynamically typed number. Before accessing the value, you must determine the detected type:
|
||||
|
||||
* `number.get_number_type()` has value `number_type::signed_integer` if we have a integer in [-9223372036854775808,9223372036854775808). You can recover the value by the `get_int64()` method applied on the `ondemand::number` instance. When `number.get_number_type()` has value `number_type::signed_integer`, you also have that `number.is_int64()` is true. Calling `get_int64()` on the `ondemand::number` instance when `number.get_number_type()` is not `number_type::signed_integer` is unsafe. You may replace `get_int64()` by a cast to a `int64_t` value.
|
||||
* `number.get_number_type()` has value `number_type::unsigned_integer` if we have a integer in [9223372036854775808,18446744073709551616). You can recover the value by the `get_uint64()` method applied on the `ondemand::number` instance. When `number.get_number_type()` has value `number_type::unsigned_integer`, you also have that `number.is_uint64()` is true. Calling `get_uint64()` on the `ondemand::number` instance when `number.get_number_type()` is not `number_type::unsigned_integer` is unsafe. You may replace `get_uint64()` by a cast to a `uint64_t` value.
|
||||
* `number.get_number_type()` has value `number_type::unsigned_integer` if we have a integer in `[9223372036854775808,18446744073709551616)`. You can recover the value by the `get_uint64()` method applied on the `ondemand::number` instance. When `number.get_number_type()` has value `number_type::unsigned_integer`, you also have that `number.is_uint64()` is true. Calling `get_uint64()` on the `ondemand::number` instance when `number.get_number_type()` is not `number_type::unsigned_integer` is unsafe. You may replace `get_uint64()` by a cast to a `uint64_t` value.
|
||||
* `number.get_number_type()` has value `number_type::floating_point_number` if we have and we have a floating-point (binary64) number. You can recover the value by the `get_double()` method applied on the `ondemand::number` instance. When `number.get_number_type()` has value `number_type::floating_point_number`, you also have that `number.is_double()` is true. Calling `get_double()` on the `ondemand::number` instance when `number.get_number_type()` is not `number_type::floating_point_number` is unsafe. You may replace `get_double()` by a cast to a `double` value.
|
||||
* When the value is an integer outside of the valid ranges for a 64-bit integers, e.g., when it is smaller than -9223372036854775808 or larger than 18446744073709551615, then `number.get_number_type()` has value `number_type::big_integer`. If you try to parse
|
||||
such a number of `get_number()`, you get the error `BIGINT_ERROR`. You can access the underlying string of digits with the function `raw_json_token()` which returns an `std::string_view` instance starting at the beginning of the digit. You can also call `get_double()` to get a floating-point approximation.
|
||||
|
||||
|
||||
You must check the type before accessing the value: it is an error to call `get_int64()` when `number.get_number_type()` is not `number_type::signed_integer` and when `number.is_int64()` is false. You are responsible for this check as the user of the library.
|
||||
@@ -1944,6 +1946,9 @@ Consider the following example:
|
||||
std::cout << "float: " << double(num) << " ";
|
||||
std::cout << "float: " << num.get_double() << std::endl;
|
||||
break;
|
||||
case ondemand::number_type::big_integer:
|
||||
std::cout << "big-integer: " << val.raw_json_token() << std::endl;
|
||||
break;
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -1959,6 +1964,55 @@ It will output:
|
||||
9999999999999999999 negative: 0 is_integer: 1 large 64-bit integer: 9999999999999999999 large 64-bit integer: 9999999999999999999
|
||||
```
|
||||
|
||||
In the following example, we have an array of integers that are outside the valid range of 64-bit signed or
|
||||
unsigned integers. Calling `get_number_type()` on the values returns `ondemand::number_type::big_integer`.
|
||||
You can try to represent these big integers as 64-bit floating-point numbers, though you typically lose
|
||||
precision in the process (as illustrated in the example).
|
||||
|
||||
```C++
|
||||
ondemand::parser parser;
|
||||
padded_string docdata = R"([-9223372036854775809, 18446744073709551617, 99999999999999999999999 ])"_padded;
|
||||
double dexpected[] = {-9223372036854775808.0, 18446744073709551616.0, 1e23};
|
||||
ondemand::document doc = parser.iterate(docdata);
|
||||
ondemand::array arr = doc.get_array();
|
||||
for(ondemand::value val : arr) {
|
||||
if(val.get_number_type() == ondemand::number_type::big_integer) {
|
||||
std::cout << val.get_double() << std::endl;
|
||||
// might print -9.22337e+18, 1.84467e+19, 1e+23
|
||||
}
|
||||
}
|
||||
```
|
||||
This program might print:
|
||||
```
|
||||
-9.22337e+18
|
||||
1.84467e+19
|
||||
1e+23
|
||||
```
|
||||
|
||||
You may get access to the underlying string representing the big integer with
|
||||
`raw_json_token()` and you may parse the resulting number strings using your own parser.
|
||||
|
||||
```c++
|
||||
ondemand::parser parser;
|
||||
padded_string docdata = R"([-9223372036854775809, 18446744073709551617, 99999999999999999999999 ])"_padded;
|
||||
ondemand::document doc = parser.iterate(docdata);
|
||||
ondemand::array arr = doc.get_array();
|
||||
for(ondemand::value val : arr) {
|
||||
// val.get_number_type() == ondemand::number_type::big_integer
|
||||
if(val.get_number_type() == ondemand::number_type::big_integer) {
|
||||
std::string_view token = val.raw_json_token();
|
||||
// token = "-9223372036854775809", "18446744073709551617", "99999999999999999999999 "
|
||||
std::cout << "'" << token << "'" << std::endl;
|
||||
}
|
||||
}
|
||||
```
|
||||
This code prints the following:
|
||||
```
|
||||
'-9223372036854775809'
|
||||
'18446744073709551617'
|
||||
'99999999999999999999999 '
|
||||
```
|
||||
|
||||
Raw Strings From Keys
|
||||
-----------
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ enum error_code {
|
||||
F_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'f'
|
||||
N_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'n'
|
||||
NUMBER_ERROR, ///< Problem while parsing a number
|
||||
BIGINT_ERROR, ///< The integer value exceeds 64 bits
|
||||
UTF8_ERROR, ///< the input is not valid UTF-8
|
||||
UNINITIALIZED, ///< unknown error, or uninitialized document
|
||||
EMPTY, ///< no structural element found
|
||||
|
||||
@@ -37,7 +37,8 @@ class dom_parser_implementation;
|
||||
enum class number_type {
|
||||
floating_point_number=1, /// a binary64 number
|
||||
signed_integer, /// a signed integer that fits in a 64-bit word using two's complement
|
||||
unsigned_integer /// a positive integer larger or equal to 1<<63
|
||||
unsigned_integer, /// a positive integer larger or equal to 1<<63
|
||||
big_integer /// a big integer that does not fit in a 64-bit word
|
||||
};
|
||||
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
|
||||
@@ -20,11 +20,13 @@ namespace numberparsing {
|
||||
#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE)))
|
||||
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE)))
|
||||
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE)))
|
||||
#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR)
|
||||
#else
|
||||
#define INVALID_NUMBER(SRC) (NUMBER_ERROR)
|
||||
#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE))
|
||||
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE))
|
||||
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE))
|
||||
#define BIGINT_NUMBER(SRC) (BIGINT_ERROR)
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
@@ -355,6 +357,10 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) {
|
||||
return true;
|
||||
}
|
||||
|
||||
simdjson_inline bool is_digit(const uint8_t c) {
|
||||
return static_cast<uint8_t>(c - '0') <= 9;
|
||||
}
|
||||
|
||||
simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
|
||||
// we continue with the fiction that we have an integer. If the
|
||||
// floating point number is representable as x * 10^z for some integer
|
||||
@@ -434,6 +440,23 @@ simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const s
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) {
|
||||
const uint8_t *const srcend = src + max_length;
|
||||
bool negative = (*src == '-'); // we can always read at least one character after the '-'
|
||||
const uint8_t *p = src + uint8_t(negative);
|
||||
if(p == srcend) { return false; }
|
||||
if(*p == '0') {
|
||||
++p;
|
||||
if(p == srcend) { return true; }
|
||||
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
|
||||
return true;
|
||||
}
|
||||
while(p != srcend && is_digit(*p)) { ++p; }
|
||||
if(p == srcend) { return true; }
|
||||
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
|
||||
return true;
|
||||
}
|
||||
|
||||
simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) {
|
||||
// It is possible that the integer had an overflow.
|
||||
// We have to handle the case where we have 0.0000somenumber.
|
||||
@@ -508,6 +531,18 @@ simdjson_inline error_code write_float(const uint8_t *const src, bool negative,
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
// parse the number at src
|
||||
// define JSON_TEST_NUMBERS for unit testing
|
||||
//
|
||||
// It is assumed that the number is followed by a structural ({,},],[) character
|
||||
// or a white space character. If that is not the case (e.g., when the JSON
|
||||
// document is made of a single number), then it is necessary to copy the
|
||||
// content and append a space before calling this function.
|
||||
//
|
||||
// Our objective is accurate parsing (ULP of 0) at high speed.
|
||||
template<typename W>
|
||||
simdjson_inline error_code parse_number(const uint8_t *const src, W &writer);
|
||||
|
||||
// for performance analysis, it is sometimes useful to skip parsing
|
||||
#ifdef SIMDJSON_SKIPNUMBERPARSING
|
||||
|
||||
@@ -586,11 +621,11 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
|
||||
// The longest positive 64-bit number is 20 digits.
|
||||
// We do it this way so we don't trigger this branch unless we must.
|
||||
size_t longest_digit_count = negative ? 19 : 20;
|
||||
if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); }
|
||||
if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); }
|
||||
if (digit_count == longest_digit_count) {
|
||||
if (negative) {
|
||||
// Anything negative above INT64_MAX+1 is invalid
|
||||
if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src); }
|
||||
if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); }
|
||||
WRITE_INTEGER(~i+1, src, writer);
|
||||
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
|
||||
return SUCCESS;
|
||||
@@ -1060,19 +1095,32 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
|
||||
src += uint8_t(negative);
|
||||
const uint8_t *p = src;
|
||||
while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
|
||||
size_t digit_count = size_t(p - src);
|
||||
if ( p == src ) { return NUMBER_ERROR; }
|
||||
if (jsoncharutils::is_structural_or_whitespace(*p)) {
|
||||
static const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
|
||||
// We have an integer.
|
||||
if(simdjson_unlikely(digit_count > 20)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
// If the number is negative and valid, it must be a signed integer.
|
||||
if(negative) { return number_type::signed_integer; }
|
||||
if(negative) {
|
||||
if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer;
|
||||
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
return number_type::signed_integer;
|
||||
}
|
||||
// Let us check if we have a big integer (>=2**64).
|
||||
static const uint8_t * two_to_sixtyfour = reinterpret_cast<const uint8_t *>("18446744073709551616");
|
||||
if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
// The number is positive and smaller than 18446744073709551616 (or 2**64).
|
||||
// We want values larger or equal to 9223372036854775808 to be unsigned
|
||||
// integers, and the other values to be signed integers.
|
||||
int digit_count = int(p - src);
|
||||
if(digit_count >= 19) {
|
||||
const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
|
||||
if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
|
||||
return number_type::unsigned_integer;
|
||||
}
|
||||
if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) {
|
||||
return number_type::unsigned_integer;
|
||||
}
|
||||
return number_type::signed_integer;
|
||||
}
|
||||
@@ -1250,6 +1298,7 @@ inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
|
||||
case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
|
||||
case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
|
||||
case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
|
||||
case number_type::big_integer: out << "big integer"; break;
|
||||
default: SIMDJSON_UNREACHABLE();
|
||||
}
|
||||
return out;
|
||||
@@ -1258,4 +1307,4 @@ inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_GENERIC_NUMBERPARSING_H
|
||||
#endif // SIMDJSON_GENERIC_NUMBERPARSING_H
|
||||
|
||||
@@ -295,7 +295,7 @@ simdjson_inline simdjson_result<number> document::get_number() noexcept {
|
||||
|
||||
simdjson_inline simdjson_result<std::string_view> document::raw_json_token() noexcept {
|
||||
auto _iter = get_root_value_iterator();
|
||||
return std::string_view(reinterpret_cast<const char*>(_iter.peek_start()), _iter.peek_start_length());
|
||||
return std::string_view(reinterpret_cast<const char*>(_iter.peek_start()), _iter.peek_root_length());
|
||||
}
|
||||
|
||||
simdjson_inline simdjson_result<value> document::at_pointer(std::string_view json_pointer) noexcept {
|
||||
@@ -535,6 +535,11 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> simdj
|
||||
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
template <class T, typename std::enable_if<std::is_same<T, SIMDJSON_IMPLEMENTATION::ondemand::document>::value == false>::type>
|
||||
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::operator T() noexcept(false) {
|
||||
if (error()) { throw simdjson_error(error()); }
|
||||
return first;
|
||||
}
|
||||
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::operator SIMDJSON_IMPLEMENTATION::ondemand::array() & noexcept(false) {
|
||||
if (error()) { throw simdjson_error(error()); }
|
||||
return first;
|
||||
@@ -842,6 +847,11 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> simdj
|
||||
return first.get_number();
|
||||
}
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
template <class T, typename std::enable_if<std::is_same<T, SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::value == false>::type>
|
||||
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator T() noexcept(false) {
|
||||
if (error()) { throw simdjson_error(error()); }
|
||||
return first;
|
||||
}
|
||||
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator SIMDJSON_IMPLEMENTATION::ondemand::array() & noexcept(false) {
|
||||
if (error()) { throw simdjson_error(error()); }
|
||||
return first;
|
||||
|
||||
@@ -179,21 +179,23 @@ public:
|
||||
* @returns INCORRECT_TYPE If the JSON value is not the given type.
|
||||
*/
|
||||
template<typename T> simdjson_inline simdjson_result<T> get() & noexcept {
|
||||
// Unless the simdjson library provides an inline implementation, calling this method should
|
||||
// Unless the simdjson library or the user provides an inline implementation, calling this method should
|
||||
// immediately fail.
|
||||
static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. "
|
||||
"The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, "
|
||||
"int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), "
|
||||
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template.");
|
||||
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."
|
||||
" You may also add support for custom types, see our documentation.");
|
||||
}
|
||||
/** @overload template<typename T> simdjson_result<T> get() & noexcept */
|
||||
template<typename T> simdjson_inline simdjson_result<T> get() && noexcept {
|
||||
// Unless the simdjson library provides an inline implementation, calling this method should
|
||||
// Unless the simdjson library or the user provides an inline implementation, calling this method should
|
||||
// immediately fail.
|
||||
static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. "
|
||||
"The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, "
|
||||
"int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), "
|
||||
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template.");
|
||||
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."
|
||||
" You may also add support for custom types, see our documentation.");
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -212,6 +214,15 @@ public:
|
||||
template<typename T> simdjson_inline error_code get(T &out) && noexcept;
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
/**
|
||||
* Cast this JSON value to an instance of type T. The programmer is responsible for
|
||||
* providing an implementation of get<T> for the type T, if T is not one of the types
|
||||
* supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.)
|
||||
*
|
||||
* See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types
|
||||
*
|
||||
* @returns An instance of type T
|
||||
*/
|
||||
template <class T>
|
||||
explicit simdjson_inline operator T() noexcept(false);
|
||||
/**
|
||||
@@ -386,7 +397,7 @@ public:
|
||||
* APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order.
|
||||
*
|
||||
* Use find_field() if you are sure fields will be in order (or are willing to treat it as if the
|
||||
* field wasn't there when they aren't).
|
||||
* field was not there when they are not in order).
|
||||
*
|
||||
* You must consume the fields on an object one at a time. A request for a new key
|
||||
* invalidates previous field values: it makes them unsafe. E.g., the array
|
||||
@@ -467,9 +478,11 @@ public:
|
||||
* get_number().get_number_type().
|
||||
*
|
||||
* get_number_type() is number_type::unsigned_integer if we have
|
||||
* an integer greater or equal to 9223372036854775808
|
||||
* an integer greater or equal to 9223372036854775808 and no larger than 18446744073709551615.
|
||||
* get_number_type() is number_type::signed_integer if we have an
|
||||
* integer that is less than 9223372036854775808
|
||||
* integer that is less than 9223372036854775808 and greater or equal to -9223372036854775808.
|
||||
* get_number_type() is number_type::big_integer if we have an integer outside
|
||||
* of those ranges (either larger than 18446744073709551615 or smaller than -9223372036854775808).
|
||||
* Otherwise, get_number_type() has value number_type::floating_point_number
|
||||
*
|
||||
* This function requires processing the number string, but it is expected
|
||||
@@ -782,7 +795,7 @@ public:
|
||||
template<typename T> simdjson_inline error_code get(T &out) & noexcept;
|
||||
template<typename T> simdjson_inline error_code get(T &out) && noexcept;
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
template <class T>
|
||||
template <class T, typename std::enable_if<std::is_same<T, SIMDJSON_IMPLEMENTATION::ondemand::document>::value == false>::type>
|
||||
explicit simdjson_inline operator T() noexcept(false);
|
||||
simdjson_inline operator SIMDJSON_IMPLEMENTATION::ondemand::array() & noexcept(false);
|
||||
simdjson_inline operator SIMDJSON_IMPLEMENTATION::ondemand::object() & noexcept(false);
|
||||
@@ -853,7 +866,7 @@ public:
|
||||
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> get_value() noexcept;
|
||||
simdjson_inline simdjson_result<bool> is_null() noexcept;
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
template <class T>
|
||||
template <class T, typename std::enable_if<std::is_same<T, SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::value == false>::type>
|
||||
explicit simdjson_inline operator T() noexcept(false);
|
||||
simdjson_inline operator SIMDJSON_IMPLEMENTATION::ondemand::array() & noexcept(false);
|
||||
simdjson_inline operator SIMDJSON_IMPLEMENTATION::ondemand::object() & noexcept(false);
|
||||
|
||||
@@ -286,6 +286,12 @@ simdjson_inline uint32_t json_iterator::peek_length(token_position position) con
|
||||
#endif // SIMDJSON_CHECK_EOF
|
||||
return token.peek_length(position);
|
||||
}
|
||||
simdjson_inline uint32_t json_iterator::peek_root_length(token_position position) const noexcept {
|
||||
#if SIMDJSON_CHECK_EOF
|
||||
assert_valid_position(position);
|
||||
#endif // SIMDJSON_CHECK_EOF
|
||||
return token.peek_root_length(position);
|
||||
}
|
||||
|
||||
simdjson_inline token_position json_iterator::last_position() const noexcept {
|
||||
// The following line fails under some compilers...
|
||||
|
||||
@@ -185,6 +185,14 @@ public:
|
||||
* @param position The position of the token to retrieve.
|
||||
*/
|
||||
simdjson_inline uint32_t peek_length(token_position position) const noexcept;
|
||||
/**
|
||||
* Get the maximum length of the JSON text for the current root token.
|
||||
*
|
||||
* The length will include any whitespace at the end of the token.
|
||||
*
|
||||
* @param position The position of the token to retrieve.
|
||||
*/
|
||||
simdjson_inline uint32_t peek_root_length(token_position position) const noexcept;
|
||||
/**
|
||||
* Get the JSON text for the last token in the document.
|
||||
*
|
||||
|
||||
@@ -48,7 +48,6 @@ simdjson_inline number::operator uint64_t() const noexcept {
|
||||
return get_uint64();
|
||||
}
|
||||
|
||||
|
||||
simdjson_inline bool number::is_int64() const noexcept {
|
||||
return get_number_type() == number_type::signed_integer;
|
||||
}
|
||||
|
||||
@@ -77,7 +77,7 @@ public:
|
||||
* APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order.
|
||||
*
|
||||
* Use find_field() if you are sure fields will be in order (or are willing to treat it as if the
|
||||
* field wasn't there when they aren't).
|
||||
* field was not there when they are not in order).
|
||||
*
|
||||
* If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful
|
||||
* that only one field is returned.
|
||||
|
||||
@@ -37,6 +37,11 @@ simdjson_inline uint32_t token_iterator::peek_length(token_position position) co
|
||||
return *(position+1) - *position;
|
||||
}
|
||||
|
||||
simdjson_inline uint32_t token_iterator::peek_root_length(token_position position) const noexcept {
|
||||
return *(position+2) - *(position) > *(position+1) - *(position) ?
|
||||
*(position+1) - *(position)
|
||||
: *(position+2) - *(position);
|
||||
}
|
||||
simdjson_inline const uint8_t *token_iterator::peek(int32_t delta) const noexcept {
|
||||
return &buf[*(_position+delta)];
|
||||
}
|
||||
|
||||
@@ -77,7 +77,14 @@ public:
|
||||
* @param position The position of the token.
|
||||
*/
|
||||
simdjson_inline uint32_t peek_length(token_position position) const noexcept;
|
||||
|
||||
/**
|
||||
* Get the maximum length of the JSON text for a root token.
|
||||
*
|
||||
* The length will include any whitespace at the end of the token.
|
||||
*
|
||||
* @param position The position of the token (start of the document).
|
||||
*/
|
||||
simdjson_inline uint32_t peek_root_length(token_position position) const noexcept;
|
||||
/**
|
||||
* Return the current index.
|
||||
*/
|
||||
|
||||
@@ -36,12 +36,13 @@ public:
|
||||
* @returns INCORRECT_TYPE If the JSON value is not the given type.
|
||||
*/
|
||||
template<typename T> simdjson_inline simdjson_result<T> get() noexcept {
|
||||
// Unless the simdjson library provides an inline implementation, calling this method should
|
||||
// Unless the simdjson library or the user provides an inline implementation, calling this method should
|
||||
// immediately fail.
|
||||
static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. "
|
||||
"The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, "
|
||||
"int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), "
|
||||
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template.");
|
||||
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."
|
||||
" You may also add support for custom types, see our documentation.");
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -196,7 +197,10 @@ public:
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
/**
|
||||
* Cast this JSON value to an instance of type T. The programmer is responsible for
|
||||
* providing an implementation of get<T> for the type T.
|
||||
* providing an implementation of get<T> for the type T, if T is not one of the types
|
||||
* supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.).
|
||||
*
|
||||
* See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types
|
||||
*
|
||||
* @returns An instance of type T
|
||||
*/
|
||||
@@ -365,7 +369,7 @@ public:
|
||||
* that only one field is returned.
|
||||
*
|
||||
* Use find_field() if you are sure fields will be in order (or are willing to treat it as if the
|
||||
* field wasn't there when they aren't).
|
||||
* field as not there when they are not in order).
|
||||
*
|
||||
* @param key The key to look up.
|
||||
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
|
||||
@@ -440,10 +444,12 @@ public:
|
||||
* get_number().get_number_type().
|
||||
*
|
||||
* get_number_type() is number_type::unsigned_integer if we have
|
||||
* an integer greater or equal to 9223372036854775808
|
||||
* an integer greater or equal to 9223372036854775808.
|
||||
* get_number_type() is number_type::signed_integer if we have an
|
||||
* integer that is less than 9223372036854775808
|
||||
* Otherwise, get_number_type() has value number_type::floating_point_number
|
||||
* integer that is less than 9223372036854775808.
|
||||
* get_number_type() is number_type::big_integer for integers that do not fit in 64 bits,
|
||||
* in which case the digit_count is set to the length of the big integer string.
|
||||
* Otherwise, get_number_type() has value number_type::floating_point_number.
|
||||
*
|
||||
* This function requires processing the number string, but it is expected
|
||||
* to be faster than get_number().get_number_type() because it is does not
|
||||
@@ -470,6 +476,8 @@ public:
|
||||
* You can recover the value by calling number.get_uint64() and you
|
||||
* have that number.is_uint64() is true.
|
||||
*
|
||||
* For integers that do not fit in 64 bits, the function returns BIGINT_ERROR error code.
|
||||
*
|
||||
* Otherwise, number.get_number_type() has value number_type::floating_point_number
|
||||
* and we have a binary64 number.
|
||||
* You can recover the value by calling number.get_double() and you
|
||||
@@ -485,7 +493,6 @@ public:
|
||||
*/
|
||||
simdjson_warn_unused simdjson_inline simdjson_result<number> get_number() noexcept;
|
||||
|
||||
|
||||
/**
|
||||
* Get the raw JSON for this token.
|
||||
*
|
||||
@@ -729,7 +736,7 @@ public:
|
||||
* APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order.
|
||||
*
|
||||
* Use find_field() if you are sure fields will be in order (or are willing to treat it as if the
|
||||
* field wasn't there when they aren't).
|
||||
* field as not there when they are not in order).
|
||||
*
|
||||
* @param key The key to look up.
|
||||
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
|
||||
|
||||
@@ -137,7 +137,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::find_
|
||||
} else if (!is_open()) {
|
||||
#if SIMDJSON_DEVELOPMENT_CHECKS
|
||||
// If we're past the end of the object, we're being iterated out of order.
|
||||
// Note: this isn't perfect detection. It's possible the user is inside some other object; if so,
|
||||
// Note: this is not perfect detection. It's possible the user is inside some other object; if so,
|
||||
// this object iterator will blithely scan that object for fields.
|
||||
if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; }
|
||||
#endif
|
||||
@@ -249,7 +249,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::find_
|
||||
|
||||
#if SIMDJSON_DEVELOPMENT_CHECKS
|
||||
// If we're past the end of the object, we're being iterated out of order.
|
||||
// Note: this isn't perfect detection. It's possible the user is inside some other object; if so,
|
||||
// Note: this is not perfect detection. It's possible the user is inside some other object; if so,
|
||||
// this object iterator will blithely scan that object for fields.
|
||||
if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; }
|
||||
#endif
|
||||
@@ -591,7 +591,7 @@ simdjson_inline simdjson_result<number> value_iterator::get_number() noexcept {
|
||||
}
|
||||
|
||||
simdjson_inline simdjson_result<bool> value_iterator::is_root_integer(bool check_trailing) noexcept {
|
||||
auto max_len = peek_start_length();
|
||||
auto max_len = peek_root_length();
|
||||
auto json = peek_root_scalar("is_root_integer");
|
||||
uint8_t tmpbuf[20+1+1]{}; // <20 digits> is the longest possible unsigned integer
|
||||
tmpbuf[20+1] = '\0'; // make sure that buffer is always null terminated.
|
||||
@@ -607,7 +607,7 @@ simdjson_inline simdjson_result<bool> value_iterator::is_root_integer(bool check
|
||||
}
|
||||
|
||||
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::number_type> value_iterator::get_root_number_type(bool check_trailing) noexcept {
|
||||
auto max_len = peek_start_length();
|
||||
auto max_len = peek_root_length();
|
||||
auto json = peek_root_scalar("number");
|
||||
// Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/,
|
||||
// 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest
|
||||
@@ -615,7 +615,12 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::number_type> value_iter
|
||||
uint8_t tmpbuf[1074+8+1+1];
|
||||
tmpbuf[1074+8+1] = '\0'; // make sure that buffer is always null terminated.
|
||||
if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074+8+1)) {
|
||||
logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters");
|
||||
if(numberparsing::check_if_integer(json, max_len)) {
|
||||
if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; }
|
||||
logger::log_error(*_json_iter, start_position(), depth(), "Found big integer");
|
||||
return number_type::big_integer;
|
||||
}
|
||||
logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer");
|
||||
return NUMBER_ERROR;
|
||||
}
|
||||
auto answer = numberparsing::get_number_type(tmpbuf);
|
||||
@@ -623,15 +628,21 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::number_type> value_iter
|
||||
return answer;
|
||||
}
|
||||
simdjson_inline simdjson_result<number> value_iterator::get_root_number(bool check_trailing) noexcept {
|
||||
auto max_len = peek_start_length();
|
||||
auto max_len = peek_root_length();
|
||||
auto json = peek_root_scalar("number");
|
||||
// Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/,
|
||||
// 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest
|
||||
// number: -0.<fraction>e-308.
|
||||
// NOTE: the current approach doesn't work for very big integer numbers containing more than 1074 digits.
|
||||
uint8_t tmpbuf[1074+8+1+1];
|
||||
tmpbuf[1074+8+1] = '\0'; // make sure that buffer is always null terminated.
|
||||
if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074+8+1)) {
|
||||
logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters");
|
||||
if(numberparsing::check_if_integer(json, max_len)) {
|
||||
if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; }
|
||||
logger::log_error(*_json_iter, start_position(), depth(), "Found big integer");
|
||||
return BIGINT_ERROR;
|
||||
}
|
||||
logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer");
|
||||
return NUMBER_ERROR;
|
||||
}
|
||||
number num;
|
||||
@@ -663,7 +674,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> value_iter
|
||||
return raw_json_string(json+1);
|
||||
}
|
||||
simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> value_iterator::get_root_uint64(bool check_trailing) noexcept {
|
||||
auto max_len = peek_start_length();
|
||||
auto max_len = peek_root_length();
|
||||
auto json = peek_root_scalar("uint64");
|
||||
uint8_t tmpbuf[20+1+1]{}; // <20 digits> is the longest possible unsigned integer
|
||||
tmpbuf[20+1] = '\0'; // make sure that buffer is always null terminated.
|
||||
@@ -679,7 +690,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> value_iterator::g
|
||||
return result;
|
||||
}
|
||||
simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> value_iterator::get_root_uint64_in_string(bool check_trailing) noexcept {
|
||||
auto max_len = peek_start_length();
|
||||
auto max_len = peek_root_length();
|
||||
auto json = peek_root_scalar("uint64");
|
||||
uint8_t tmpbuf[20+1+1]{}; // <20 digits> is the longest possible unsigned integer
|
||||
tmpbuf[20+1] = '\0'; // make sure that buffer is always null terminated.
|
||||
@@ -695,7 +706,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> value_iterator::g
|
||||
return result;
|
||||
}
|
||||
simdjson_warn_unused simdjson_inline simdjson_result<int64_t> value_iterator::get_root_int64(bool check_trailing) noexcept {
|
||||
auto max_len = peek_start_length();
|
||||
auto max_len = peek_root_length();
|
||||
auto json = peek_root_scalar("int64");
|
||||
uint8_t tmpbuf[20+1+1]; // -<19 digits> is the longest possible integer
|
||||
tmpbuf[20+1] = '\0'; // make sure that buffer is always null terminated.
|
||||
@@ -712,7 +723,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<int64_t> value_iterator::ge
|
||||
return result;
|
||||
}
|
||||
simdjson_warn_unused simdjson_inline simdjson_result<int64_t> value_iterator::get_root_int64_in_string(bool check_trailing) noexcept {
|
||||
auto max_len = peek_start_length();
|
||||
auto max_len = peek_root_length();
|
||||
auto json = peek_root_scalar("int64");
|
||||
uint8_t tmpbuf[20+1+1]; // -<19 digits> is the longest possible integer
|
||||
tmpbuf[20+1] = '\0'; // make sure that buffer is always null terminated.
|
||||
@@ -729,7 +740,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<int64_t> value_iterator::ge
|
||||
return result;
|
||||
}
|
||||
simdjson_warn_unused simdjson_inline simdjson_result<double> value_iterator::get_root_double(bool check_trailing) noexcept {
|
||||
auto max_len = peek_start_length();
|
||||
auto max_len = peek_root_length();
|
||||
auto json = peek_root_scalar("double");
|
||||
// Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/,
|
||||
// 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest
|
||||
@@ -749,7 +760,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<double> value_iterator::get
|
||||
}
|
||||
|
||||
simdjson_warn_unused simdjson_inline simdjson_result<double> value_iterator::get_root_double_in_string(bool check_trailing) noexcept {
|
||||
auto max_len = peek_start_length();
|
||||
auto max_len = peek_root_length();
|
||||
auto json = peek_root_scalar("double");
|
||||
// Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/,
|
||||
// 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest
|
||||
@@ -768,7 +779,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<double> value_iterator::get
|
||||
return result;
|
||||
}
|
||||
simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::get_root_bool(bool check_trailing) noexcept {
|
||||
auto max_len = peek_start_length();
|
||||
auto max_len = peek_root_length();
|
||||
auto json = peek_root_scalar("bool");
|
||||
uint8_t tmpbuf[5+1+1]; // +1 for null termination
|
||||
tmpbuf[5+1] = '\0'; // make sure that buffer is always null terminated.
|
||||
@@ -781,7 +792,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::get_r
|
||||
return result;
|
||||
}
|
||||
simdjson_inline simdjson_result<bool> value_iterator::is_root_null(bool check_trailing) noexcept {
|
||||
auto max_len = peek_start_length();
|
||||
auto max_len = peek_root_length();
|
||||
auto json = peek_root_scalar("null");
|
||||
bool result = (max_len >= 4 && !atomparsing::str4ncmp(json, "null") &&
|
||||
(max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[4])));
|
||||
@@ -853,6 +864,9 @@ simdjson_inline const uint8_t *value_iterator::peek_start() const noexcept {
|
||||
simdjson_inline uint32_t value_iterator::peek_start_length() const noexcept {
|
||||
return _json_iter->peek_length(start_position());
|
||||
}
|
||||
simdjson_inline uint32_t value_iterator::peek_root_length() const noexcept {
|
||||
return _json_iter->peek_root_length(start_position());
|
||||
}
|
||||
|
||||
simdjson_inline const uint8_t *value_iterator::peek_scalar(const char *type) noexcept {
|
||||
logger::log_value(*_json_iter, start_position(), depth(), type);
|
||||
|
||||
@@ -375,6 +375,7 @@ protected:
|
||||
simdjson_inline simdjson_result<bool> parse_bool(const uint8_t *json) const noexcept;
|
||||
simdjson_inline const uint8_t *peek_start() const noexcept;
|
||||
simdjson_inline uint32_t peek_start_length() const noexcept;
|
||||
simdjson_inline uint32_t peek_root_length() const noexcept;
|
||||
|
||||
/**
|
||||
* The general idea of the advance_... methods and the peek_* methods
|
||||
|
||||
@@ -38,6 +38,8 @@
|
||||
#define SIMDJSON_IS_ARM64 1
|
||||
#elif defined(__riscv) && __riscv_xlen == 64
|
||||
#define SIMDJSON_IS_RISCV64 1
|
||||
#elif defined(__loongarch_lp64)
|
||||
#define SIMDJSON_IS_LOONGARCH64 1
|
||||
#elif defined(__PPC64__) || defined(_M_PPC64)
|
||||
#if defined(__ALTIVEC__)
|
||||
#define SIMDJSON_IS_PPC64_VMX 1
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
#define SIMDJSON_SIMDJSON_VERSION_H
|
||||
|
||||
/** The version of simdjson being used (major.minor.revision) */
|
||||
#define SIMDJSON_VERSION "3.7.0"
|
||||
#define SIMDJSON_VERSION "3.8.0"
|
||||
|
||||
namespace simdjson {
|
||||
enum {
|
||||
@@ -15,7 +15,7 @@ enum {
|
||||
/**
|
||||
* The minor version (major.MINOR.revision) of simdjson being used.
|
||||
*/
|
||||
SIMDJSON_VERSION_MINOR = 7,
|
||||
SIMDJSON_VERSION_MINOR = 8,
|
||||
/**
|
||||
* The revision (major.minor.REVISION) of simdjson being used.
|
||||
*/
|
||||
|
||||
+389
-85
@@ -1,4 +1,4 @@
|
||||
/* auto-generated on 2024-02-18 11:50:12 -0500. Do not edit! */
|
||||
/* auto-generated on 2024-03-10 13:24:07 -0400. Do not edit! */
|
||||
/* including simdjson.cpp: */
|
||||
/* begin file simdjson.cpp */
|
||||
#define SIMDJSON_SRC_SIMDJSON_CPP
|
||||
@@ -111,6 +111,8 @@
|
||||
#define SIMDJSON_IS_ARM64 1
|
||||
#elif defined(__riscv) && __riscv_xlen == 64
|
||||
#define SIMDJSON_IS_RISCV64 1
|
||||
#elif defined(__loongarch_lp64)
|
||||
#define SIMDJSON_IS_LOONGARCH64 1
|
||||
#elif defined(__PPC64__) || defined(_M_PPC64)
|
||||
#if defined(__ALTIVEC__)
|
||||
#define SIMDJSON_IS_PPC64_VMX 1
|
||||
@@ -2345,6 +2347,7 @@ enum error_code {
|
||||
F_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'f'
|
||||
N_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'n'
|
||||
NUMBER_ERROR, ///< Problem while parsing a number
|
||||
BIGINT_ERROR, ///< The integer value exceeds 64 bits
|
||||
UTF8_ERROR, ///< the input is not valid UTF-8
|
||||
UNINITIALIZED, ///< unknown error, or uninitialized document
|
||||
EMPTY, ///< no structural element found
|
||||
@@ -4493,6 +4496,7 @@ namespace internal {
|
||||
{ F_ATOM_ERROR, "F_ATOM_ERROR: Problem while parsing an atom starting with the letter 'f'" },
|
||||
{ N_ATOM_ERROR, "N_ATOM_ERROR: Problem while parsing an atom starting with the letter 'n'" },
|
||||
{ NUMBER_ERROR, "NUMBER_ERROR: Problem while parsing a number" },
|
||||
{ BIGINT_ERROR, "BIGINT_ERROR: Big integer value that cannot be represented using 64 bits" },
|
||||
{ UTF8_ERROR, "UTF8_ERROR: The input is not valid UTF-8" },
|
||||
{ UNINITIALIZED, "UNINITIALIZED: Uninitialized" },
|
||||
{ EMPTY, "EMPTY: no JSON found" },
|
||||
@@ -8213,7 +8217,8 @@ class dom_parser_implementation;
|
||||
enum class number_type {
|
||||
floating_point_number=1, /// a binary64 number
|
||||
signed_integer, /// a signed integer that fits in a 64-bit word using two's complement
|
||||
unsigned_integer /// a positive integer larger or equal to 1<<63
|
||||
unsigned_integer, /// a positive integer larger or equal to 1<<63
|
||||
big_integer /// a big integer that does not fit in a 64-bit word
|
||||
};
|
||||
|
||||
} // namespace arm64
|
||||
@@ -8661,11 +8666,13 @@ namespace numberparsing {
|
||||
#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE)))
|
||||
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE)))
|
||||
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE)))
|
||||
#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR)
|
||||
#else
|
||||
#define INVALID_NUMBER(SRC) (NUMBER_ERROR)
|
||||
#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE))
|
||||
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE))
|
||||
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE))
|
||||
#define BIGINT_NUMBER(SRC) (BIGINT_ERROR)
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
@@ -8996,6 +9003,10 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) {
|
||||
return true;
|
||||
}
|
||||
|
||||
simdjson_inline bool is_digit(const uint8_t c) {
|
||||
return static_cast<uint8_t>(c - '0') <= 9;
|
||||
}
|
||||
|
||||
simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
|
||||
// we continue with the fiction that we have an integer. If the
|
||||
// floating point number is representable as x * 10^z for some integer
|
||||
@@ -9075,6 +9086,23 @@ simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const s
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) {
|
||||
const uint8_t *const srcend = src + max_length;
|
||||
bool negative = (*src == '-'); // we can always read at least one character after the '-'
|
||||
const uint8_t *p = src + uint8_t(negative);
|
||||
if(p == srcend) { return false; }
|
||||
if(*p == '0') {
|
||||
++p;
|
||||
if(p == srcend) { return true; }
|
||||
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
|
||||
return true;
|
||||
}
|
||||
while(p != srcend && is_digit(*p)) { ++p; }
|
||||
if(p == srcend) { return true; }
|
||||
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
|
||||
return true;
|
||||
}
|
||||
|
||||
simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) {
|
||||
// It is possible that the integer had an overflow.
|
||||
// We have to handle the case where we have 0.0000somenumber.
|
||||
@@ -9149,6 +9177,18 @@ simdjson_inline error_code write_float(const uint8_t *const src, bool negative,
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
// parse the number at src
|
||||
// define JSON_TEST_NUMBERS for unit testing
|
||||
//
|
||||
// It is assumed that the number is followed by a structural ({,},],[) character
|
||||
// or a white space character. If that is not the case (e.g., when the JSON
|
||||
// document is made of a single number), then it is necessary to copy the
|
||||
// content and append a space before calling this function.
|
||||
//
|
||||
// Our objective is accurate parsing (ULP of 0) at high speed.
|
||||
template<typename W>
|
||||
simdjson_inline error_code parse_number(const uint8_t *const src, W &writer);
|
||||
|
||||
// for performance analysis, it is sometimes useful to skip parsing
|
||||
#ifdef SIMDJSON_SKIPNUMBERPARSING
|
||||
|
||||
@@ -9227,11 +9267,11 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
|
||||
// The longest positive 64-bit number is 20 digits.
|
||||
// We do it this way so we don't trigger this branch unless we must.
|
||||
size_t longest_digit_count = negative ? 19 : 20;
|
||||
if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); }
|
||||
if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); }
|
||||
if (digit_count == longest_digit_count) {
|
||||
if (negative) {
|
||||
// Anything negative above INT64_MAX+1 is invalid
|
||||
if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src); }
|
||||
if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); }
|
||||
WRITE_INTEGER(~i+1, src, writer);
|
||||
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
|
||||
return SUCCESS;
|
||||
@@ -9701,19 +9741,32 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
|
||||
src += uint8_t(negative);
|
||||
const uint8_t *p = src;
|
||||
while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
|
||||
size_t digit_count = size_t(p - src);
|
||||
if ( p == src ) { return NUMBER_ERROR; }
|
||||
if (jsoncharutils::is_structural_or_whitespace(*p)) {
|
||||
static const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
|
||||
// We have an integer.
|
||||
if(simdjson_unlikely(digit_count > 20)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
// If the number is negative and valid, it must be a signed integer.
|
||||
if(negative) { return number_type::signed_integer; }
|
||||
if(negative) {
|
||||
if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer;
|
||||
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
return number_type::signed_integer;
|
||||
}
|
||||
// Let us check if we have a big integer (>=2**64).
|
||||
static const uint8_t * two_to_sixtyfour = reinterpret_cast<const uint8_t *>("18446744073709551616");
|
||||
if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
// The number is positive and smaller than 18446744073709551616 (or 2**64).
|
||||
// We want values larger or equal to 9223372036854775808 to be unsigned
|
||||
// integers, and the other values to be signed integers.
|
||||
int digit_count = int(p - src);
|
||||
if(digit_count >= 19) {
|
||||
const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
|
||||
if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
|
||||
return number_type::unsigned_integer;
|
||||
}
|
||||
if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) {
|
||||
return number_type::unsigned_integer;
|
||||
}
|
||||
return number_type::signed_integer;
|
||||
}
|
||||
@@ -9891,6 +9944,7 @@ inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
|
||||
case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
|
||||
case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
|
||||
case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
|
||||
case number_type::big_integer: out << "big integer"; break;
|
||||
default: SIMDJSON_UNREACHABLE();
|
||||
}
|
||||
return out;
|
||||
@@ -13019,8 +13073,8 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
|
||||
// Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD)
|
||||
constexpr uint32_t substitution_code_point = 0xfffd;
|
||||
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
|
||||
// conversion isn't valid; we defer the check for this to inside the
|
||||
// multilingual plane check
|
||||
// conversion is not valid; we defer the check for this to inside the
|
||||
// multilingual plane check.
|
||||
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
|
||||
*src_ptr += 6;
|
||||
|
||||
@@ -13077,8 +13131,8 @@ simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr,
|
||||
// It is not ideal that this function is nearly identical to handle_unicode_codepoint.
|
||||
//
|
||||
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
|
||||
// conversion isn't valid; we defer the check for this to inside the
|
||||
// multilingual plane check
|
||||
// conversion is not valid; we defer the check for this to inside the
|
||||
// multilingual plane check.
|
||||
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
|
||||
*src_ptr += 6;
|
||||
// If we found a high surrogate, we must
|
||||
@@ -14501,7 +14555,8 @@ class dom_parser_implementation;
|
||||
enum class number_type {
|
||||
floating_point_number=1, /// a binary64 number
|
||||
signed_integer, /// a signed integer that fits in a 64-bit word using two's complement
|
||||
unsigned_integer /// a positive integer larger or equal to 1<<63
|
||||
unsigned_integer, /// a positive integer larger or equal to 1<<63
|
||||
big_integer /// a big integer that does not fit in a 64-bit word
|
||||
};
|
||||
|
||||
} // namespace haswell
|
||||
@@ -14949,11 +15004,13 @@ namespace numberparsing {
|
||||
#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE)))
|
||||
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE)))
|
||||
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE)))
|
||||
#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR)
|
||||
#else
|
||||
#define INVALID_NUMBER(SRC) (NUMBER_ERROR)
|
||||
#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE))
|
||||
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE))
|
||||
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE))
|
||||
#define BIGINT_NUMBER(SRC) (BIGINT_ERROR)
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
@@ -15284,6 +15341,10 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) {
|
||||
return true;
|
||||
}
|
||||
|
||||
simdjson_inline bool is_digit(const uint8_t c) {
|
||||
return static_cast<uint8_t>(c - '0') <= 9;
|
||||
}
|
||||
|
||||
simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
|
||||
// we continue with the fiction that we have an integer. If the
|
||||
// floating point number is representable as x * 10^z for some integer
|
||||
@@ -15363,6 +15424,23 @@ simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const s
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) {
|
||||
const uint8_t *const srcend = src + max_length;
|
||||
bool negative = (*src == '-'); // we can always read at least one character after the '-'
|
||||
const uint8_t *p = src + uint8_t(negative);
|
||||
if(p == srcend) { return false; }
|
||||
if(*p == '0') {
|
||||
++p;
|
||||
if(p == srcend) { return true; }
|
||||
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
|
||||
return true;
|
||||
}
|
||||
while(p != srcend && is_digit(*p)) { ++p; }
|
||||
if(p == srcend) { return true; }
|
||||
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
|
||||
return true;
|
||||
}
|
||||
|
||||
simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) {
|
||||
// It is possible that the integer had an overflow.
|
||||
// We have to handle the case where we have 0.0000somenumber.
|
||||
@@ -15437,6 +15515,18 @@ simdjson_inline error_code write_float(const uint8_t *const src, bool negative,
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
// parse the number at src
|
||||
// define JSON_TEST_NUMBERS for unit testing
|
||||
//
|
||||
// It is assumed that the number is followed by a structural ({,},],[) character
|
||||
// or a white space character. If that is not the case (e.g., when the JSON
|
||||
// document is made of a single number), then it is necessary to copy the
|
||||
// content and append a space before calling this function.
|
||||
//
|
||||
// Our objective is accurate parsing (ULP of 0) at high speed.
|
||||
template<typename W>
|
||||
simdjson_inline error_code parse_number(const uint8_t *const src, W &writer);
|
||||
|
||||
// for performance analysis, it is sometimes useful to skip parsing
|
||||
#ifdef SIMDJSON_SKIPNUMBERPARSING
|
||||
|
||||
@@ -15515,11 +15605,11 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
|
||||
// The longest positive 64-bit number is 20 digits.
|
||||
// We do it this way so we don't trigger this branch unless we must.
|
||||
size_t longest_digit_count = negative ? 19 : 20;
|
||||
if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); }
|
||||
if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); }
|
||||
if (digit_count == longest_digit_count) {
|
||||
if (negative) {
|
||||
// Anything negative above INT64_MAX+1 is invalid
|
||||
if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src); }
|
||||
if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); }
|
||||
WRITE_INTEGER(~i+1, src, writer);
|
||||
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
|
||||
return SUCCESS;
|
||||
@@ -15989,19 +16079,32 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
|
||||
src += uint8_t(negative);
|
||||
const uint8_t *p = src;
|
||||
while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
|
||||
size_t digit_count = size_t(p - src);
|
||||
if ( p == src ) { return NUMBER_ERROR; }
|
||||
if (jsoncharutils::is_structural_or_whitespace(*p)) {
|
||||
static const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
|
||||
// We have an integer.
|
||||
if(simdjson_unlikely(digit_count > 20)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
// If the number is negative and valid, it must be a signed integer.
|
||||
if(negative) { return number_type::signed_integer; }
|
||||
if(negative) {
|
||||
if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer;
|
||||
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
return number_type::signed_integer;
|
||||
}
|
||||
// Let us check if we have a big integer (>=2**64).
|
||||
static const uint8_t * two_to_sixtyfour = reinterpret_cast<const uint8_t *>("18446744073709551616");
|
||||
if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
// The number is positive and smaller than 18446744073709551616 (or 2**64).
|
||||
// We want values larger or equal to 9223372036854775808 to be unsigned
|
||||
// integers, and the other values to be signed integers.
|
||||
int digit_count = int(p - src);
|
||||
if(digit_count >= 19) {
|
||||
const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
|
||||
if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
|
||||
return number_type::unsigned_integer;
|
||||
}
|
||||
if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) {
|
||||
return number_type::unsigned_integer;
|
||||
}
|
||||
return number_type::signed_integer;
|
||||
}
|
||||
@@ -16179,6 +16282,7 @@ inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
|
||||
case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
|
||||
case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
|
||||
case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
|
||||
case number_type::big_integer: out << "big integer"; break;
|
||||
default: SIMDJSON_UNREACHABLE();
|
||||
}
|
||||
return out;
|
||||
@@ -19184,8 +19288,8 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
|
||||
// Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD)
|
||||
constexpr uint32_t substitution_code_point = 0xfffd;
|
||||
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
|
||||
// conversion isn't valid; we defer the check for this to inside the
|
||||
// multilingual plane check
|
||||
// conversion is not valid; we defer the check for this to inside the
|
||||
// multilingual plane check.
|
||||
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
|
||||
*src_ptr += 6;
|
||||
|
||||
@@ -19242,8 +19346,8 @@ simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr,
|
||||
// It is not ideal that this function is nearly identical to handle_unicode_codepoint.
|
||||
//
|
||||
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
|
||||
// conversion isn't valid; we defer the check for this to inside the
|
||||
// multilingual plane check
|
||||
// conversion is not valid; we defer the check for this to inside the
|
||||
// multilingual plane check.
|
||||
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
|
||||
*src_ptr += 6;
|
||||
// If we found a high surrogate, we must
|
||||
@@ -20661,7 +20765,8 @@ class dom_parser_implementation;
|
||||
enum class number_type {
|
||||
floating_point_number=1, /// a binary64 number
|
||||
signed_integer, /// a signed integer that fits in a 64-bit word using two's complement
|
||||
unsigned_integer /// a positive integer larger or equal to 1<<63
|
||||
unsigned_integer, /// a positive integer larger or equal to 1<<63
|
||||
big_integer /// a big integer that does not fit in a 64-bit word
|
||||
};
|
||||
|
||||
} // namespace icelake
|
||||
@@ -21109,11 +21214,13 @@ namespace numberparsing {
|
||||
#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE)))
|
||||
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE)))
|
||||
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE)))
|
||||
#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR)
|
||||
#else
|
||||
#define INVALID_NUMBER(SRC) (NUMBER_ERROR)
|
||||
#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE))
|
||||
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE))
|
||||
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE))
|
||||
#define BIGINT_NUMBER(SRC) (BIGINT_ERROR)
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
@@ -21444,6 +21551,10 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) {
|
||||
return true;
|
||||
}
|
||||
|
||||
simdjson_inline bool is_digit(const uint8_t c) {
|
||||
return static_cast<uint8_t>(c - '0') <= 9;
|
||||
}
|
||||
|
||||
simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
|
||||
// we continue with the fiction that we have an integer. If the
|
||||
// floating point number is representable as x * 10^z for some integer
|
||||
@@ -21523,6 +21634,23 @@ simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const s
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) {
|
||||
const uint8_t *const srcend = src + max_length;
|
||||
bool negative = (*src == '-'); // we can always read at least one character after the '-'
|
||||
const uint8_t *p = src + uint8_t(negative);
|
||||
if(p == srcend) { return false; }
|
||||
if(*p == '0') {
|
||||
++p;
|
||||
if(p == srcend) { return true; }
|
||||
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
|
||||
return true;
|
||||
}
|
||||
while(p != srcend && is_digit(*p)) { ++p; }
|
||||
if(p == srcend) { return true; }
|
||||
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
|
||||
return true;
|
||||
}
|
||||
|
||||
simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) {
|
||||
// It is possible that the integer had an overflow.
|
||||
// We have to handle the case where we have 0.0000somenumber.
|
||||
@@ -21597,6 +21725,18 @@ simdjson_inline error_code write_float(const uint8_t *const src, bool negative,
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
// parse the number at src
|
||||
// define JSON_TEST_NUMBERS for unit testing
|
||||
//
|
||||
// It is assumed that the number is followed by a structural ({,},],[) character
|
||||
// or a white space character. If that is not the case (e.g., when the JSON
|
||||
// document is made of a single number), then it is necessary to copy the
|
||||
// content and append a space before calling this function.
|
||||
//
|
||||
// Our objective is accurate parsing (ULP of 0) at high speed.
|
||||
template<typename W>
|
||||
simdjson_inline error_code parse_number(const uint8_t *const src, W &writer);
|
||||
|
||||
// for performance analysis, it is sometimes useful to skip parsing
|
||||
#ifdef SIMDJSON_SKIPNUMBERPARSING
|
||||
|
||||
@@ -21675,11 +21815,11 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
|
||||
// The longest positive 64-bit number is 20 digits.
|
||||
// We do it this way so we don't trigger this branch unless we must.
|
||||
size_t longest_digit_count = negative ? 19 : 20;
|
||||
if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); }
|
||||
if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); }
|
||||
if (digit_count == longest_digit_count) {
|
||||
if (negative) {
|
||||
// Anything negative above INT64_MAX+1 is invalid
|
||||
if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src); }
|
||||
if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); }
|
||||
WRITE_INTEGER(~i+1, src, writer);
|
||||
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
|
||||
return SUCCESS;
|
||||
@@ -22149,19 +22289,32 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
|
||||
src += uint8_t(negative);
|
||||
const uint8_t *p = src;
|
||||
while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
|
||||
size_t digit_count = size_t(p - src);
|
||||
if ( p == src ) { return NUMBER_ERROR; }
|
||||
if (jsoncharutils::is_structural_or_whitespace(*p)) {
|
||||
static const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
|
||||
// We have an integer.
|
||||
if(simdjson_unlikely(digit_count > 20)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
// If the number is negative and valid, it must be a signed integer.
|
||||
if(negative) { return number_type::signed_integer; }
|
||||
if(negative) {
|
||||
if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer;
|
||||
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
return number_type::signed_integer;
|
||||
}
|
||||
// Let us check if we have a big integer (>=2**64).
|
||||
static const uint8_t * two_to_sixtyfour = reinterpret_cast<const uint8_t *>("18446744073709551616");
|
||||
if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
// The number is positive and smaller than 18446744073709551616 (or 2**64).
|
||||
// We want values larger or equal to 9223372036854775808 to be unsigned
|
||||
// integers, and the other values to be signed integers.
|
||||
int digit_count = int(p - src);
|
||||
if(digit_count >= 19) {
|
||||
const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
|
||||
if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
|
||||
return number_type::unsigned_integer;
|
||||
}
|
||||
if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) {
|
||||
return number_type::unsigned_integer;
|
||||
}
|
||||
return number_type::signed_integer;
|
||||
}
|
||||
@@ -22339,6 +22492,7 @@ inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
|
||||
case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
|
||||
case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
|
||||
case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
|
||||
case number_type::big_integer: out << "big integer"; break;
|
||||
default: SIMDJSON_UNREACHABLE();
|
||||
}
|
||||
return out;
|
||||
@@ -25342,8 +25496,8 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
|
||||
// Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD)
|
||||
constexpr uint32_t substitution_code_point = 0xfffd;
|
||||
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
|
||||
// conversion isn't valid; we defer the check for this to inside the
|
||||
// multilingual plane check
|
||||
// conversion is not valid; we defer the check for this to inside the
|
||||
// multilingual plane check.
|
||||
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
|
||||
*src_ptr += 6;
|
||||
|
||||
@@ -25400,8 +25554,8 @@ simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr,
|
||||
// It is not ideal that this function is nearly identical to handle_unicode_codepoint.
|
||||
//
|
||||
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
|
||||
// conversion isn't valid; we defer the check for this to inside the
|
||||
// multilingual plane check
|
||||
// conversion is not valid; we defer the check for this to inside the
|
||||
// multilingual plane check.
|
||||
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
|
||||
*src_ptr += 6;
|
||||
// If we found a high surrogate, we must
|
||||
@@ -26977,7 +27131,8 @@ class dom_parser_implementation;
|
||||
enum class number_type {
|
||||
floating_point_number=1, /// a binary64 number
|
||||
signed_integer, /// a signed integer that fits in a 64-bit word using two's complement
|
||||
unsigned_integer /// a positive integer larger or equal to 1<<63
|
||||
unsigned_integer, /// a positive integer larger or equal to 1<<63
|
||||
big_integer /// a big integer that does not fit in a 64-bit word
|
||||
};
|
||||
|
||||
} // namespace ppc64
|
||||
@@ -27425,11 +27580,13 @@ namespace numberparsing {
|
||||
#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE)))
|
||||
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE)))
|
||||
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE)))
|
||||
#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR)
|
||||
#else
|
||||
#define INVALID_NUMBER(SRC) (NUMBER_ERROR)
|
||||
#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE))
|
||||
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE))
|
||||
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE))
|
||||
#define BIGINT_NUMBER(SRC) (BIGINT_ERROR)
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
@@ -27760,6 +27917,10 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) {
|
||||
return true;
|
||||
}
|
||||
|
||||
simdjson_inline bool is_digit(const uint8_t c) {
|
||||
return static_cast<uint8_t>(c - '0') <= 9;
|
||||
}
|
||||
|
||||
simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
|
||||
// we continue with the fiction that we have an integer. If the
|
||||
// floating point number is representable as x * 10^z for some integer
|
||||
@@ -27839,6 +28000,23 @@ simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const s
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) {
|
||||
const uint8_t *const srcend = src + max_length;
|
||||
bool negative = (*src == '-'); // we can always read at least one character after the '-'
|
||||
const uint8_t *p = src + uint8_t(negative);
|
||||
if(p == srcend) { return false; }
|
||||
if(*p == '0') {
|
||||
++p;
|
||||
if(p == srcend) { return true; }
|
||||
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
|
||||
return true;
|
||||
}
|
||||
while(p != srcend && is_digit(*p)) { ++p; }
|
||||
if(p == srcend) { return true; }
|
||||
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
|
||||
return true;
|
||||
}
|
||||
|
||||
simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) {
|
||||
// It is possible that the integer had an overflow.
|
||||
// We have to handle the case where we have 0.0000somenumber.
|
||||
@@ -27913,6 +28091,18 @@ simdjson_inline error_code write_float(const uint8_t *const src, bool negative,
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
// parse the number at src
|
||||
// define JSON_TEST_NUMBERS for unit testing
|
||||
//
|
||||
// It is assumed that the number is followed by a structural ({,},],[) character
|
||||
// or a white space character. If that is not the case (e.g., when the JSON
|
||||
// document is made of a single number), then it is necessary to copy the
|
||||
// content and append a space before calling this function.
|
||||
//
|
||||
// Our objective is accurate parsing (ULP of 0) at high speed.
|
||||
template<typename W>
|
||||
simdjson_inline error_code parse_number(const uint8_t *const src, W &writer);
|
||||
|
||||
// for performance analysis, it is sometimes useful to skip parsing
|
||||
#ifdef SIMDJSON_SKIPNUMBERPARSING
|
||||
|
||||
@@ -27991,11 +28181,11 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
|
||||
// The longest positive 64-bit number is 20 digits.
|
||||
// We do it this way so we don't trigger this branch unless we must.
|
||||
size_t longest_digit_count = negative ? 19 : 20;
|
||||
if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); }
|
||||
if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); }
|
||||
if (digit_count == longest_digit_count) {
|
||||
if (negative) {
|
||||
// Anything negative above INT64_MAX+1 is invalid
|
||||
if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src); }
|
||||
if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); }
|
||||
WRITE_INTEGER(~i+1, src, writer);
|
||||
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
|
||||
return SUCCESS;
|
||||
@@ -28465,19 +28655,32 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
|
||||
src += uint8_t(negative);
|
||||
const uint8_t *p = src;
|
||||
while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
|
||||
size_t digit_count = size_t(p - src);
|
||||
if ( p == src ) { return NUMBER_ERROR; }
|
||||
if (jsoncharutils::is_structural_or_whitespace(*p)) {
|
||||
static const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
|
||||
// We have an integer.
|
||||
if(simdjson_unlikely(digit_count > 20)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
// If the number is negative and valid, it must be a signed integer.
|
||||
if(negative) { return number_type::signed_integer; }
|
||||
if(negative) {
|
||||
if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer;
|
||||
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
return number_type::signed_integer;
|
||||
}
|
||||
// Let us check if we have a big integer (>=2**64).
|
||||
static const uint8_t * two_to_sixtyfour = reinterpret_cast<const uint8_t *>("18446744073709551616");
|
||||
if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
// The number is positive and smaller than 18446744073709551616 (or 2**64).
|
||||
// We want values larger or equal to 9223372036854775808 to be unsigned
|
||||
// integers, and the other values to be signed integers.
|
||||
int digit_count = int(p - src);
|
||||
if(digit_count >= 19) {
|
||||
const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
|
||||
if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
|
||||
return number_type::unsigned_integer;
|
||||
}
|
||||
if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) {
|
||||
return number_type::unsigned_integer;
|
||||
}
|
||||
return number_type::signed_integer;
|
||||
}
|
||||
@@ -28655,6 +28858,7 @@ inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
|
||||
case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
|
||||
case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
|
||||
case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
|
||||
case number_type::big_integer: out << "big integer"; break;
|
||||
default: SIMDJSON_UNREACHABLE();
|
||||
}
|
||||
return out;
|
||||
@@ -31771,8 +31975,8 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
|
||||
// Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD)
|
||||
constexpr uint32_t substitution_code_point = 0xfffd;
|
||||
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
|
||||
// conversion isn't valid; we defer the check for this to inside the
|
||||
// multilingual plane check
|
||||
// conversion is not valid; we defer the check for this to inside the
|
||||
// multilingual plane check.
|
||||
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
|
||||
*src_ptr += 6;
|
||||
|
||||
@@ -31829,8 +32033,8 @@ simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr,
|
||||
// It is not ideal that this function is nearly identical to handle_unicode_codepoint.
|
||||
//
|
||||
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
|
||||
// conversion isn't valid; we defer the check for this to inside the
|
||||
// multilingual plane check
|
||||
// conversion is not valid; we defer the check for this to inside the
|
||||
// multilingual plane check.
|
||||
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
|
||||
*src_ptr += 6;
|
||||
// If we found a high surrogate, we must
|
||||
@@ -33659,7 +33863,8 @@ class dom_parser_implementation;
|
||||
enum class number_type {
|
||||
floating_point_number=1, /// a binary64 number
|
||||
signed_integer, /// a signed integer that fits in a 64-bit word using two's complement
|
||||
unsigned_integer /// a positive integer larger or equal to 1<<63
|
||||
unsigned_integer, /// a positive integer larger or equal to 1<<63
|
||||
big_integer /// a big integer that does not fit in a 64-bit word
|
||||
};
|
||||
|
||||
} // namespace westmere
|
||||
@@ -34107,11 +34312,13 @@ namespace numberparsing {
|
||||
#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE)))
|
||||
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE)))
|
||||
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE)))
|
||||
#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR)
|
||||
#else
|
||||
#define INVALID_NUMBER(SRC) (NUMBER_ERROR)
|
||||
#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE))
|
||||
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE))
|
||||
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE))
|
||||
#define BIGINT_NUMBER(SRC) (BIGINT_ERROR)
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
@@ -34442,6 +34649,10 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) {
|
||||
return true;
|
||||
}
|
||||
|
||||
simdjson_inline bool is_digit(const uint8_t c) {
|
||||
return static_cast<uint8_t>(c - '0') <= 9;
|
||||
}
|
||||
|
||||
simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
|
||||
// we continue with the fiction that we have an integer. If the
|
||||
// floating point number is representable as x * 10^z for some integer
|
||||
@@ -34521,6 +34732,23 @@ simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const s
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) {
|
||||
const uint8_t *const srcend = src + max_length;
|
||||
bool negative = (*src == '-'); // we can always read at least one character after the '-'
|
||||
const uint8_t *p = src + uint8_t(negative);
|
||||
if(p == srcend) { return false; }
|
||||
if(*p == '0') {
|
||||
++p;
|
||||
if(p == srcend) { return true; }
|
||||
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
|
||||
return true;
|
||||
}
|
||||
while(p != srcend && is_digit(*p)) { ++p; }
|
||||
if(p == srcend) { return true; }
|
||||
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
|
||||
return true;
|
||||
}
|
||||
|
||||
simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) {
|
||||
// It is possible that the integer had an overflow.
|
||||
// We have to handle the case where we have 0.0000somenumber.
|
||||
@@ -34595,6 +34823,18 @@ simdjson_inline error_code write_float(const uint8_t *const src, bool negative,
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
// parse the number at src
|
||||
// define JSON_TEST_NUMBERS for unit testing
|
||||
//
|
||||
// It is assumed that the number is followed by a structural ({,},],[) character
|
||||
// or a white space character. If that is not the case (e.g., when the JSON
|
||||
// document is made of a single number), then it is necessary to copy the
|
||||
// content and append a space before calling this function.
|
||||
//
|
||||
// Our objective is accurate parsing (ULP of 0) at high speed.
|
||||
template<typename W>
|
||||
simdjson_inline error_code parse_number(const uint8_t *const src, W &writer);
|
||||
|
||||
// for performance analysis, it is sometimes useful to skip parsing
|
||||
#ifdef SIMDJSON_SKIPNUMBERPARSING
|
||||
|
||||
@@ -34673,11 +34913,11 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
|
||||
// The longest positive 64-bit number is 20 digits.
|
||||
// We do it this way so we don't trigger this branch unless we must.
|
||||
size_t longest_digit_count = negative ? 19 : 20;
|
||||
if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); }
|
||||
if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); }
|
||||
if (digit_count == longest_digit_count) {
|
||||
if (negative) {
|
||||
// Anything negative above INT64_MAX+1 is invalid
|
||||
if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src); }
|
||||
if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); }
|
||||
WRITE_INTEGER(~i+1, src, writer);
|
||||
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
|
||||
return SUCCESS;
|
||||
@@ -35147,19 +35387,32 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
|
||||
src += uint8_t(negative);
|
||||
const uint8_t *p = src;
|
||||
while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
|
||||
size_t digit_count = size_t(p - src);
|
||||
if ( p == src ) { return NUMBER_ERROR; }
|
||||
if (jsoncharutils::is_structural_or_whitespace(*p)) {
|
||||
static const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
|
||||
// We have an integer.
|
||||
if(simdjson_unlikely(digit_count > 20)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
// If the number is negative and valid, it must be a signed integer.
|
||||
if(negative) { return number_type::signed_integer; }
|
||||
if(negative) {
|
||||
if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer;
|
||||
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
return number_type::signed_integer;
|
||||
}
|
||||
// Let us check if we have a big integer (>=2**64).
|
||||
static const uint8_t * two_to_sixtyfour = reinterpret_cast<const uint8_t *>("18446744073709551616");
|
||||
if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
// The number is positive and smaller than 18446744073709551616 (or 2**64).
|
||||
// We want values larger or equal to 9223372036854775808 to be unsigned
|
||||
// integers, and the other values to be signed integers.
|
||||
int digit_count = int(p - src);
|
||||
if(digit_count >= 19) {
|
||||
const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
|
||||
if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
|
||||
return number_type::unsigned_integer;
|
||||
}
|
||||
if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) {
|
||||
return number_type::unsigned_integer;
|
||||
}
|
||||
return number_type::signed_integer;
|
||||
}
|
||||
@@ -35337,6 +35590,7 @@ inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
|
||||
case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
|
||||
case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
|
||||
case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
|
||||
case number_type::big_integer: out << "big integer"; break;
|
||||
default: SIMDJSON_UNREACHABLE();
|
||||
}
|
||||
return out;
|
||||
@@ -38774,8 +39028,8 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
|
||||
// Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD)
|
||||
constexpr uint32_t substitution_code_point = 0xfffd;
|
||||
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
|
||||
// conversion isn't valid; we defer the check for this to inside the
|
||||
// multilingual plane check
|
||||
// conversion is not valid; we defer the check for this to inside the
|
||||
// multilingual plane check.
|
||||
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
|
||||
*src_ptr += 6;
|
||||
|
||||
@@ -38832,8 +39086,8 @@ simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr,
|
||||
// It is not ideal that this function is nearly identical to handle_unicode_codepoint.
|
||||
//
|
||||
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
|
||||
// conversion isn't valid; we defer the check for this to inside the
|
||||
// multilingual plane check
|
||||
// conversion is not valid; we defer the check for this to inside the
|
||||
// multilingual plane check.
|
||||
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
|
||||
*src_ptr += 6;
|
||||
// If we found a high surrogate, we must
|
||||
@@ -39768,7 +40022,8 @@ class dom_parser_implementation;
|
||||
enum class number_type {
|
||||
floating_point_number=1, /// a binary64 number
|
||||
signed_integer, /// a signed integer that fits in a 64-bit word using two's complement
|
||||
unsigned_integer /// a positive integer larger or equal to 1<<63
|
||||
unsigned_integer, /// a positive integer larger or equal to 1<<63
|
||||
big_integer /// a big integer that does not fit in a 64-bit word
|
||||
};
|
||||
|
||||
} // namespace fallback
|
||||
@@ -40216,11 +40471,13 @@ namespace numberparsing {
|
||||
#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE)))
|
||||
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE)))
|
||||
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE)))
|
||||
#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR)
|
||||
#else
|
||||
#define INVALID_NUMBER(SRC) (NUMBER_ERROR)
|
||||
#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE))
|
||||
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE))
|
||||
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE))
|
||||
#define BIGINT_NUMBER(SRC) (BIGINT_ERROR)
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
@@ -40551,6 +40808,10 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) {
|
||||
return true;
|
||||
}
|
||||
|
||||
simdjson_inline bool is_digit(const uint8_t c) {
|
||||
return static_cast<uint8_t>(c - '0') <= 9;
|
||||
}
|
||||
|
||||
simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
|
||||
// we continue with the fiction that we have an integer. If the
|
||||
// floating point number is representable as x * 10^z for some integer
|
||||
@@ -40630,6 +40891,23 @@ simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const s
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) {
|
||||
const uint8_t *const srcend = src + max_length;
|
||||
bool negative = (*src == '-'); // we can always read at least one character after the '-'
|
||||
const uint8_t *p = src + uint8_t(negative);
|
||||
if(p == srcend) { return false; }
|
||||
if(*p == '0') {
|
||||
++p;
|
||||
if(p == srcend) { return true; }
|
||||
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
|
||||
return true;
|
||||
}
|
||||
while(p != srcend && is_digit(*p)) { ++p; }
|
||||
if(p == srcend) { return true; }
|
||||
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
|
||||
return true;
|
||||
}
|
||||
|
||||
simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) {
|
||||
// It is possible that the integer had an overflow.
|
||||
// We have to handle the case where we have 0.0000somenumber.
|
||||
@@ -40704,6 +40982,18 @@ simdjson_inline error_code write_float(const uint8_t *const src, bool negative,
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
// parse the number at src
|
||||
// define JSON_TEST_NUMBERS for unit testing
|
||||
//
|
||||
// It is assumed that the number is followed by a structural ({,},],[) character
|
||||
// or a white space character. If that is not the case (e.g., when the JSON
|
||||
// document is made of a single number), then it is necessary to copy the
|
||||
// content and append a space before calling this function.
|
||||
//
|
||||
// Our objective is accurate parsing (ULP of 0) at high speed.
|
||||
template<typename W>
|
||||
simdjson_inline error_code parse_number(const uint8_t *const src, W &writer);
|
||||
|
||||
// for performance analysis, it is sometimes useful to skip parsing
|
||||
#ifdef SIMDJSON_SKIPNUMBERPARSING
|
||||
|
||||
@@ -40782,11 +41072,11 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
|
||||
// The longest positive 64-bit number is 20 digits.
|
||||
// We do it this way so we don't trigger this branch unless we must.
|
||||
size_t longest_digit_count = negative ? 19 : 20;
|
||||
if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); }
|
||||
if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); }
|
||||
if (digit_count == longest_digit_count) {
|
||||
if (negative) {
|
||||
// Anything negative above INT64_MAX+1 is invalid
|
||||
if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src); }
|
||||
if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); }
|
||||
WRITE_INTEGER(~i+1, src, writer);
|
||||
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
|
||||
return SUCCESS;
|
||||
@@ -41256,19 +41546,32 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
|
||||
src += uint8_t(negative);
|
||||
const uint8_t *p = src;
|
||||
while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
|
||||
size_t digit_count = size_t(p - src);
|
||||
if ( p == src ) { return NUMBER_ERROR; }
|
||||
if (jsoncharutils::is_structural_or_whitespace(*p)) {
|
||||
static const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
|
||||
// We have an integer.
|
||||
if(simdjson_unlikely(digit_count > 20)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
// If the number is negative and valid, it must be a signed integer.
|
||||
if(negative) { return number_type::signed_integer; }
|
||||
if(negative) {
|
||||
if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer;
|
||||
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
return number_type::signed_integer;
|
||||
}
|
||||
// Let us check if we have a big integer (>=2**64).
|
||||
static const uint8_t * two_to_sixtyfour = reinterpret_cast<const uint8_t *>("18446744073709551616");
|
||||
if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) {
|
||||
return number_type::big_integer;
|
||||
}
|
||||
// The number is positive and smaller than 18446744073709551616 (or 2**64).
|
||||
// We want values larger or equal to 9223372036854775808 to be unsigned
|
||||
// integers, and the other values to be signed integers.
|
||||
int digit_count = int(p - src);
|
||||
if(digit_count >= 19) {
|
||||
const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
|
||||
if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
|
||||
return number_type::unsigned_integer;
|
||||
}
|
||||
if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) {
|
||||
return number_type::unsigned_integer;
|
||||
}
|
||||
return number_type::signed_integer;
|
||||
}
|
||||
@@ -41446,6 +41749,7 @@ inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
|
||||
case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
|
||||
case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
|
||||
case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
|
||||
case number_type::big_integer: out << "big integer"; break;
|
||||
default: SIMDJSON_UNREACHABLE();
|
||||
}
|
||||
return out;
|
||||
@@ -41966,8 +42270,8 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
|
||||
// Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD)
|
||||
constexpr uint32_t substitution_code_point = 0xfffd;
|
||||
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
|
||||
// conversion isn't valid; we defer the check for this to inside the
|
||||
// multilingual plane check
|
||||
// conversion is not valid; we defer the check for this to inside the
|
||||
// multilingual plane check.
|
||||
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
|
||||
*src_ptr += 6;
|
||||
|
||||
@@ -42024,8 +42328,8 @@ simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr,
|
||||
// It is not ideal that this function is nearly identical to handle_unicode_codepoint.
|
||||
//
|
||||
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
|
||||
// conversion isn't valid; we defer the check for this to inside the
|
||||
// multilingual plane check
|
||||
// conversion is not valid; we defer the check for this to inside the
|
||||
// multilingual plane check.
|
||||
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
|
||||
*src_ptr += 6;
|
||||
// If we found a high surrogate, we must
|
||||
|
||||
+1008
-285
File diff suppressed because it is too large
Load Diff
@@ -53,8 +53,8 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
|
||||
// Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD)
|
||||
constexpr uint32_t substitution_code_point = 0xfffd;
|
||||
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
|
||||
// conversion isn't valid; we defer the check for this to inside the
|
||||
// multilingual plane check
|
||||
// conversion is not valid; we defer the check for this to inside the
|
||||
// multilingual plane check.
|
||||
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
|
||||
*src_ptr += 6;
|
||||
|
||||
@@ -111,8 +111,8 @@ simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr,
|
||||
// It is not ideal that this function is nearly identical to handle_unicode_codepoint.
|
||||
//
|
||||
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
|
||||
// conversion isn't valid; we defer the check for this to inside the
|
||||
// multilingual plane check
|
||||
// conversion is not valid; we defer the check for this to inside the
|
||||
// multilingual plane check.
|
||||
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
|
||||
*src_ptr += 6;
|
||||
// If we found a high surrogate, we must
|
||||
|
||||
@@ -18,6 +18,7 @@ namespace internal {
|
||||
{ F_ATOM_ERROR, "F_ATOM_ERROR: Problem while parsing an atom starting with the letter 'f'" },
|
||||
{ N_ATOM_ERROR, "N_ATOM_ERROR: Problem while parsing an atom starting with the letter 'n'" },
|
||||
{ NUMBER_ERROR, "NUMBER_ERROR: Problem while parsing a number" },
|
||||
{ BIGINT_ERROR, "BIGINT_ERROR: Big integer value that cannot be represented using 64 bits" },
|
||||
{ UTF8_ERROR, "UTF8_ERROR: The input is not valid UTF-8" },
|
||||
{ UNINITIALIZED, "UNINITIALIZED: Uninitialized" },
|
||||
{ EMPTY, "EMPTY: no JSON found" },
|
||||
|
||||
@@ -102,7 +102,7 @@ namespace array_error_tests {
|
||||
ONDEMAND_SUBTEST("unclosed extra comma", R"({ "a": [,)", assert_iterate(doc["a"], { INCOMPLETE_ARRAY_OR_OBJECT }));
|
||||
ONDEMAND_SUBTEST("unclosed extra comma", R"({ "a": [,,)", assert_iterate(doc["a"], { INCOMPLETE_ARRAY_OR_OBJECT }));
|
||||
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1 )", assert_iterate(doc["a"], { INCOMPLETE_ARRAY_OR_OBJECT }));
|
||||
// TODO These pass the user values that may run past the end of the buffer if they aren't careful
|
||||
// TODO These pass the user values that may run past the end of the buffer if they are not careful
|
||||
// In particular, if the padding is decorated with the wrong values, we could cause overrun!
|
||||
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1,)", assert_iterate(doc["a"], { INCOMPLETE_ARRAY_OR_OBJECT }));
|
||||
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1)", assert_iterate(doc["a"], { INCOMPLETE_ARRAY_OR_OBJECT }));
|
||||
|
||||
@@ -651,6 +651,26 @@ namespace document_stream_tests {
|
||||
}
|
||||
|
||||
|
||||
bool issue2137() {
|
||||
TEST_START();
|
||||
auto json = "true { "_padded;
|
||||
ondemand::parser parser;
|
||||
ondemand::document_stream stream;
|
||||
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
||||
for (auto doc: stream) {
|
||||
bool val{};
|
||||
ASSERT_SUCCESS(doc.get_bool().get(val));
|
||||
std::string_view raw_json;
|
||||
ASSERT_SUCCESS(doc.raw_json_token().get(raw_json));
|
||||
ASSERT_EQUAL(val, 1);
|
||||
std::string s(raw_json);
|
||||
ASSERT_EQUAL(s, "true ");
|
||||
}
|
||||
size_t t = stream.truncated_bytes();
|
||||
ASSERT_EQUAL(3, t);
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
bool issue1668() {
|
||||
TEST_START();
|
||||
auto json = R"([1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100])"_padded;
|
||||
@@ -861,6 +881,7 @@ namespace document_stream_tests {
|
||||
|
||||
bool run() {
|
||||
return
|
||||
issue2137() &&
|
||||
skipbom() &&
|
||||
issue1977() &&
|
||||
string_with_trailing() &&
|
||||
|
||||
@@ -106,6 +106,29 @@ namespace error_tests {
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
// This is a compile-only test
|
||||
struct Class {
|
||||
Class(std::string text)
|
||||
: parser()
|
||||
, doc(parser.iterate(text))
|
||||
{
|
||||
}
|
||||
simdjson::ondemand::parser parser;
|
||||
simdjson::ondemand::document doc;
|
||||
};
|
||||
bool document_in_class() {
|
||||
TEST_START();
|
||||
std::string text = "{}";
|
||||
Class c(text);
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
bool direct_document() {
|
||||
TEST_START();
|
||||
std::string text = "{}";
|
||||
simdjson::ondemand::parser parser;
|
||||
simdjson::ondemand::document doc(parser.iterate(text));
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
bool raw_json_string_except() {
|
||||
TEST_START();
|
||||
ondemand::parser parser;
|
||||
@@ -381,6 +404,8 @@ namespace error_tests {
|
||||
issue1834() &&
|
||||
issue1834_2() &&
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
document_in_class() &&
|
||||
direct_document() &&
|
||||
raw_json_string_except() &&
|
||||
raw_json_string_except_with_io() &&
|
||||
#endif
|
||||
|
||||
@@ -211,7 +211,7 @@ namespace number_tests {
|
||||
bool get_number_tests() {
|
||||
TEST_START();
|
||||
ondemand::parser parser;
|
||||
padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999,-9223372036854775807,-9223372036854775808])"_padded;
|
||||
padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999,-9223372036854775807,-9223372036854775808,12345678901234567890123])"_padded;
|
||||
ondemand::number_type expectedtypes[] = {ondemand::number_type::floating_point_number,
|
||||
ondemand::number_type::signed_integer,
|
||||
ondemand::number_type::signed_integer,
|
||||
@@ -219,10 +219,11 @@ namespace number_tests {
|
||||
ondemand::number_type::signed_integer,
|
||||
ondemand::number_type::unsigned_integer,
|
||||
ondemand::number_type::signed_integer,
|
||||
ondemand::number_type::signed_integer
|
||||
ondemand::number_type::signed_integer,
|
||||
ondemand::number_type::big_integer
|
||||
};
|
||||
bool is_negative[] = {false, false, false, false, true, false, true, true};
|
||||
bool is_integer[] = {false, true, true, false, true, true, true, true};
|
||||
bool is_negative[] = {false, false, false, false, true, false, true, true, false};
|
||||
bool is_integer[] = {false, true, true, false, true, true, true, true, true};
|
||||
|
||||
ondemand::document doc;
|
||||
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
||||
@@ -233,17 +234,25 @@ namespace number_tests {
|
||||
ondemand::value val;
|
||||
ASSERT_SUCCESS(valr.get(val));
|
||||
ondemand::number_type nt{};
|
||||
ASSERT_SUCCESS(val.get_number_type().get(nt));
|
||||
auto r = val.get_number_type().get(nt);
|
||||
ASSERT_SUCCESS(r);
|
||||
ASSERT_EQUAL(expectedtypes[counter], nt);
|
||||
ondemand::number num;
|
||||
ASSERT_SUCCESS(val.get_number().get(num));
|
||||
ASSERT_EQUAL(is_negative[counter], val.is_negative());
|
||||
bool intvalue{};
|
||||
ASSERT_SUCCESS(val.is_integer().get(intvalue));
|
||||
auto res = val.is_integer().get(intvalue);
|
||||
ASSERT_SUCCESS(res);
|
||||
ondemand::number num;
|
||||
auto rr = val.get_number().get(num);
|
||||
if(counter == 8) {
|
||||
ASSERT_EQUAL(rr, BIGINT_ERROR);
|
||||
ASSERT_EQUAL(nt, ondemand::number_type::big_integer);
|
||||
} else {
|
||||
ASSERT_SUCCESS(rr);
|
||||
ondemand::number_type t = num.get_number_type();
|
||||
ASSERT_EQUAL(expectedtypes[counter], t);
|
||||
}
|
||||
ASSERT_EQUAL(is_integer[counter], intvalue);
|
||||
ondemand::number_type t = num.get_number_type();
|
||||
ASSERT_EQUAL(expectedtypes[counter], t);
|
||||
switch(t) {
|
||||
ASSERT_EQUAL(is_negative[counter], val.is_negative());
|
||||
switch(nt) {
|
||||
case ondemand::number_type::signed_integer:
|
||||
ASSERT_TRUE(num.is_int64());
|
||||
break;
|
||||
@@ -253,25 +262,39 @@ namespace number_tests {
|
||||
case ondemand::number_type::floating_point_number:
|
||||
ASSERT_TRUE(num.is_double());
|
||||
break;
|
||||
case ondemand::number_type::big_integer:
|
||||
break;
|
||||
}
|
||||
if(counter == 0) {
|
||||
ASSERT_EQUAL(num.get_double(), 1.0);
|
||||
ASSERT_EQUAL((double)num, 1.0);
|
||||
} else if(counter == 1) {
|
||||
ASSERT_EQUAL(num.get_int64(), 3);
|
||||
ASSERT_EQUAL((int64_t)num, 3);
|
||||
} else if(counter == 2) {
|
||||
ASSERT_EQUAL(num.get_int64(), 1);
|
||||
ASSERT_EQUAL((int64_t)num, 1);
|
||||
} else if(counter == 3) {
|
||||
ASSERT_EQUAL(num.get_double(), 3.1415);
|
||||
ASSERT_EQUAL((double)num, 3.1415);
|
||||
} else if(counter == 4) {
|
||||
ASSERT_EQUAL(num.get_int64(), -13231232);
|
||||
ASSERT_EQUAL((int64_t)num, -13231232);
|
||||
} else if(counter == 5) {
|
||||
ASSERT_EQUAL(num.get_uint64(), UINT64_C(9999999999999999999));
|
||||
ASSERT_EQUAL((uint64_t)num, UINT64_C(9999999999999999999));
|
||||
switch(counter) {
|
||||
case 0:
|
||||
ASSERT_EQUAL(num.get_double(), 1.0);
|
||||
ASSERT_EQUAL((double)num, 1.0);
|
||||
break;
|
||||
case 1:
|
||||
ASSERT_EQUAL(num.get_int64(), 3);
|
||||
ASSERT_EQUAL((int64_t)num, 3);
|
||||
break;
|
||||
case 2:
|
||||
ASSERT_EQUAL(num.get_int64(), 1);
|
||||
ASSERT_EQUAL((int64_t)num, 1);
|
||||
break;
|
||||
case 3:
|
||||
ASSERT_EQUAL(num.get_double(), 3.1415);
|
||||
ASSERT_EQUAL((double)num, 3.1415);
|
||||
break;
|
||||
case 4:
|
||||
ASSERT_EQUAL(num.get_int64(), -13231232);
|
||||
ASSERT_EQUAL((int64_t)num, -13231232);
|
||||
break;
|
||||
case 5:
|
||||
ASSERT_EQUAL(num.get_uint64(), UINT64_C(9999999999999999999));
|
||||
ASSERT_EQUAL((uint64_t)num, UINT64_C(9999999999999999999));
|
||||
break;
|
||||
case 8:
|
||||
ASSERT_EQUAL(val.raw_json_token(), "12345678901234567890123");
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
counter++;
|
||||
}
|
||||
@@ -441,8 +464,50 @@ namespace number_tests {
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
bool big_int_not_zero() {
|
||||
TEST_START();
|
||||
ondemand::parser parser;
|
||||
ondemand::document doc;
|
||||
// This is not a big integer, it is a mistake
|
||||
padded_string docdata = R"(09500000000000000000000000000000000000)"_padded;
|
||||
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
||||
ASSERT_ERROR(doc.get_number(), NUMBER_ERROR);
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
bool negative_big_int() {
|
||||
TEST_START();
|
||||
ondemand::parser parser;
|
||||
ondemand::document doc;
|
||||
// This is not a big integer, it is a mistake
|
||||
padded_string docdata = R"(-18446744073709551616)"_padded;
|
||||
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
||||
ASSERT_ERROR(doc.get_number(), BIGINT_ERROR);
|
||||
std::string_view my_big;
|
||||
ASSERT_SUCCESS(doc.raw_json_token().get(my_big));
|
||||
ASSERT_EQUAL(my_big, "-18446744073709551616");
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
bool gigantic_big_int() {
|
||||
TEST_START();
|
||||
ondemand::parser parser;
|
||||
ondemand::document doc;
|
||||
std::string number(2000, '1');
|
||||
// This is not a big integer, it is a mistake
|
||||
ASSERT_SUCCESS(parser.iterate(number).get(doc));
|
||||
ASSERT_ERROR(doc.get_number(), BIGINT_ERROR);
|
||||
std::string_view my_big;
|
||||
ASSERT_SUCCESS(doc.raw_json_token().get(my_big));
|
||||
ASSERT_EQUAL(my_big, number);
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
bool run() {
|
||||
return issue2099() &&
|
||||
return gigantic_big_int() &&
|
||||
big_int_not_zero() &&
|
||||
negative_big_int() &&
|
||||
issue2099() &&
|
||||
issue2093() &&
|
||||
issue2045() &&
|
||||
issue2017() &&
|
||||
|
||||
@@ -119,7 +119,7 @@ namespace object_error_tests {
|
||||
bool object_lookup_miss_unclosed_error() {
|
||||
TEST_START();
|
||||
ONDEMAND_SUBTEST("unclosed", R"({ "a": 1, )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT));
|
||||
// TODO These next two pass the user a value that may run past the end of the buffer if they aren't careful.
|
||||
// TODO These next two pass the user a value that may run past the end of the buffer if they are not careful.
|
||||
// In particular, if the padding is decorated with the wrong values, we could cause overrun!
|
||||
ONDEMAND_SUBTEST("unclosed", R"({ "a": 1 )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT));
|
||||
ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT));
|
||||
|
||||
@@ -3,6 +3,9 @@
|
||||
#if __cpp_lib_optional >= 201606L
|
||||
#include <optional>
|
||||
#endif
|
||||
#if SIMDJSON_CPLUSPLUS17
|
||||
#include <charconv>
|
||||
#endif
|
||||
using namespace std;
|
||||
using namespace simdjson;
|
||||
using error_code=simdjson::error_code;
|
||||
@@ -382,22 +385,84 @@ bool examplecrt_realloc() {
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
#if SIMDJSON_CPLUSPLUS17
|
||||
bool big_int_array() {
|
||||
TEST_START();
|
||||
ondemand::parser parser;
|
||||
padded_string docdata = R"([-9223372036854775809, 18446744073709551617, 99999999999999999999999 ])"_padded;
|
||||
std::string expected[] = {"-9223372036854775809", "18446744073709551617", "99999999999999999999999 "};
|
||||
ondemand::document doc = parser.iterate(docdata);
|
||||
ondemand::array arr = doc.get_array();
|
||||
size_t i = 0;
|
||||
for(ondemand::value val : arr) {
|
||||
if(i > 3) {
|
||||
std::cerr << "unexpected number of elements" << std::endl;
|
||||
return false;
|
||||
}
|
||||
if(val.get_number_type() != ondemand::number_type::big_integer) {
|
||||
std::cerr << "unexpected number type" << std::endl;
|
||||
std::cout << val.get_number_type() << std::endl;
|
||||
std::cout << val.raw_json_token() << std::endl;
|
||||
return false;
|
||||
}
|
||||
std::string_view token = val.raw_json_token();
|
||||
std::string_view expected_token = expected[i];
|
||||
if(token != expected_token) {
|
||||
std::cerr << "unexpected token: " << token << " expected: " << expected_token << std::endl;
|
||||
return false;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
if(i != 3) {
|
||||
std::cerr << "unexpected number of elements" << std::endl;
|
||||
return false;
|
||||
}
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
#endif
|
||||
|
||||
bool big_int_array_as_double() {
|
||||
TEST_START();
|
||||
ondemand::parser parser;
|
||||
padded_string docdata = R"([-9223372036854775809, 18446744073709551617, 99999999999999999999999 ])"_padded;
|
||||
double dexpected[] = {-9223372036854775808.0, 18446744073709551616.0, 1e23};
|
||||
ondemand::document doc = parser.iterate(docdata);
|
||||
ondemand::array arr = doc.get_array();
|
||||
size_t i = 0;
|
||||
for(ondemand::value val : arr) {
|
||||
if(i > 3) {
|
||||
std::cerr << "unexpected number of elements" << std::endl;
|
||||
return false;
|
||||
}
|
||||
if((val.get_number_type() != ondemand::number_type::big_integer) || (dexpected[i] != val.get_double())) {
|
||||
return false;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
if(i != 3) {
|
||||
std::cerr << "unexpected number of elements" << std::endl;
|
||||
return false;
|
||||
}
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
bool number_tests() {
|
||||
TEST_START();
|
||||
ondemand::parser parser;
|
||||
padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999])"_padded;
|
||||
padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999,12345678901234567890123])"_padded;
|
||||
ondemand::document doc = parser.iterate(docdata);
|
||||
ondemand::array arr = doc.get_array();
|
||||
for(ondemand::value val : arr) {
|
||||
std::cout << val << " ";
|
||||
std::cout << "negative: " << val.is_negative() << " ";
|
||||
std::cout << "is_integer: " << val.is_integer() << " ";
|
||||
ondemand::number num = val.get_number();
|
||||
ondemand::number_type t = num.get_number_type();
|
||||
// direct computation without materializing the number:
|
||||
ondemand::number_type dt = val.get_number_type();
|
||||
if(t != dt) { throw std::runtime_error("bug"); }
|
||||
switch(t) {
|
||||
ondemand::number num;
|
||||
auto res = val.get_number().get(num);
|
||||
ondemand::number_type t = num.get_number_type();
|
||||
if(t != dt && res != BIGINT_ERROR) { throw std::runtime_error("bug"); }
|
||||
switch(dt) {
|
||||
case ondemand::number_type::signed_integer:
|
||||
std::cout << "integer: " << int64_t(num) << " ";
|
||||
std::cout << "integer: " << num.get_int64() << std::endl;
|
||||
@@ -410,6 +475,9 @@ bool number_tests() {
|
||||
std::cout << "float: " << double(num) << " ";
|
||||
std::cout << "float: " << num.get_double() << std::endl;
|
||||
break;
|
||||
case ondemand::number_type::big_integer:
|
||||
std::cout << "big-integer: " << val.raw_json_token() << std::endl;
|
||||
break;
|
||||
}
|
||||
}
|
||||
TEST_SUCCEED();
|
||||
@@ -1778,6 +1846,10 @@ bool value_raw_json_object() {
|
||||
bool run() {
|
||||
return true
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
#if SIMDJSON_CPLUSPLUS17
|
||||
&& big_int_array()
|
||||
#endif // SIMDJSON_CPLUSPLUS17
|
||||
&& big_int_array_as_double()
|
||||
&& key_raw_json_token()
|
||||
&& to_optional()
|
||||
&& value_raw_json_array() && value_raw_json_object()
|
||||
@@ -1785,7 +1857,7 @@ bool run() {
|
||||
&& at_end()
|
||||
&& example1956() && example1958()
|
||||
&& allow_comma_separated_example()
|
||||
// && basics_1() // Fails because twitter.json isn't in current directory. Compile test only.
|
||||
// && basics_1() // Fails because twitter.json is not in current directory. Compile test only.
|
||||
&& basics_treewalk()
|
||||
&& basics_treewalk_breakline()
|
||||
&& json_value_with_array_count()
|
||||
|
||||
+7
-7
@@ -16,7 +16,7 @@ def colored(r, g, b, text):
|
||||
return "\033[38;2;{};{};{}m{} \033[38;2;255;255;255m".format(r, g, b, text)
|
||||
|
||||
def extractnumbers(s):
|
||||
return tuple(map(int,re.findall("(\d+)\.(\d+)\.(\d+)",str(s))[0]))
|
||||
return tuple(map(int,re.findall(r"(\d+)\.(\d+)\.(\d+)",str(s))[0]))
|
||||
|
||||
def toversionstring(major, minor, rev):
|
||||
return str(major)+"."+str(minor)+"."+str(rev)
|
||||
@@ -123,7 +123,7 @@ newversionstring = str(newversion[0]) + "." + str(newversion[1]) + "." + str(new
|
||||
cmakefile = maindir + os.sep + "CMakeLists.txt"
|
||||
|
||||
sonumber = None
|
||||
pattern = re.compile("set\(SIMDJSON_LIB_SOVERSION \"(\d+)\" CACHE STRING \"simdjson library soversion\"\)")
|
||||
pattern = re.compile(r"set\(SIMDJSON_LIB_SOVERSION \"(\d+)\" CACHE STRING \"simdjson library soversion\"\)")
|
||||
with open (cmakefile, 'rt') as myfile:
|
||||
for line in myfile:
|
||||
m = pattern.search(line)
|
||||
@@ -138,9 +138,9 @@ if(atleastminor):
|
||||
sonumber += 1
|
||||
|
||||
for line in fileinput.input(cmakefile, inplace=1, backup='.bak'):
|
||||
line = re.sub(' VERSION \d+\.\d+\.\d+',' VERSION '+newmajorversionstring+'.'+mewminorversionstring+'.'+newrevversionstring, line.rstrip())
|
||||
line = re.sub('SIMDJSON_LIB_VERSION "\d+','SIMDJSON_LIB_VERSION "'+str(sonumber), line)
|
||||
line = re.sub('set\(SIMDJSON_LIB_SOVERSION \"\d+\"','set(SIMDJSON_LIB_SOVERSION \"'+str(sonumber)+'\"', line)
|
||||
line = re.sub(r' VERSION \d+\.\d+\.\d+',' VERSION '+newmajorversionstring+'.'+mewminorversionstring+'.'+newrevversionstring, line.rstrip())
|
||||
line = re.sub(r'SIMDJSON_LIB_VERSION "\d+','SIMDJSON_LIB_VERSION "'+str(sonumber), line)
|
||||
line = re.sub(r'set\(SIMDJSON_LIB_SOVERSION \"\d+\"','set(SIMDJSON_LIB_SOVERSION \"'+str(sonumber)+'\"', line)
|
||||
print(line)
|
||||
|
||||
print("modified "+cmakefile+", a backup was made")
|
||||
@@ -148,7 +148,7 @@ print("modified "+cmakefile+", a backup was made")
|
||||
|
||||
doxyfile = maindir + os.sep + "Doxyfile"
|
||||
for line in fileinput.input(doxyfile, inplace=1, backup='.bak'):
|
||||
line = re.sub('PROJECT_NUMBER = "\d+\.\d+\.\d+','PROJECT_NUMBER = "'+newversionstring, line.rstrip())
|
||||
line = re.sub(r'PROJECT_NUMBER = "\d+\.\d+\.\d+','PROJECT_NUMBER = "'+newversionstring, line.rstrip())
|
||||
print(line)
|
||||
print("modified "+doxyfile+", a backup was made")
|
||||
|
||||
@@ -168,7 +168,7 @@ if(cp.returncode != 0):
|
||||
print("Failed to run doxygen")
|
||||
|
||||
|
||||
pattern = re.compile("https://simdjson.org/api/(\d+\.\d+\.\d+)/index.html")
|
||||
pattern = re.compile(r"https://simdjson.org/api/(\d+\.\d+\.\d+)/index.html")
|
||||
readmefile = maindir + os.sep + "README.md"
|
||||
readmedata = open(readmefile).read()
|
||||
m = pattern.search(readmedata)
|
||||
|
||||
Reference in New Issue
Block a user