Compare commits

...

14 Commits

Author SHA1 Message Date
Daniel Lemire e1c6a778f8 3.8.0 2024-03-10 13:25:53 -04:00
Daniel Lemire d413dd5b0e v3.7.1 2024-03-10 13:24:07 -04:00
Daniel Lemire a33fdc2770 Unconfusing clang (#2147)
* In some cases, clang might try to cast an ondemand::document to an
ondemand::document, instead of calling the move constructor. So we
can disable the template cast.

* In some cases, clang might get confused when constructing an ondemand
document in a constructor: instead of calling the move constructor, it
somehow ends up trying to cast a document to a document. We can easily
disallow this behavior with std::enable_if.

* making compatible with C++11

---------

Co-authored-by: Daniel Lemire <dlemire@lemire.me>
2024-03-10 13:20:39 -04:00
Daniel Lemire 87f5f7a250 added some documentation in the code related to custom types (#2146) 2024-03-09 18:45:05 -05:00
Daniel Lemire 9923d27994 upper case 2024-03-06 16:01:21 -05:00
Piotr Rżysko 876be23545 Add link to simdjson-java (#2145) 2024-03-06 15:55:16 -05:00
Daniel Lemire 2028bdaeac This documents the big-int feature, and adds a few tests. (#2142)
* This documents the big-int feature, and adds a few tests.

* moving check_if_integer

* trimming the example.

* More trimming.

---------

Co-authored-by: Daniel Lemire <dlemire@lemire.me>
2024-02-26 20:39:20 -05:00
Daniel Lemire 1ac3a00ca9 verify issue 2137 (#2138)
* fix issue 2137

* updating the single header

---------

Co-authored-by: Daniel Lemire <dlemire@lemire.me>
2024-02-26 20:39:08 -05:00
Daniel Lemire 6d7cc2c7c1 fixing build 2024-02-26 20:00:17 -05:00
Daniel Lemire 61a52287fc Fixing apple builds 2024-02-26 19:50:22 -05:00
Daniel Lemire 37a9aaad48 [no ci] tweaking the wording 2024-02-26 14:05:12 -05:00
Serge Aleynikov 78325888ea Add functionality to detect big integers (#2139) 2024-02-26 14:00:33 -05:00
Shi Pujin a90cb5d900 add loongarch64 (#2141) 2024-02-26 13:54:49 -05:00
Daniel Lemire 34a01ec3af removing unnecessary spaces 2024-02-23 19:21:31 -05:00
35 changed files with 1875 additions and 485 deletions
+7 -1
View File
@@ -103,6 +103,12 @@
"charconv": "cpp",
"source_location": "cpp",
"strstream": "cpp",
"typeindex": "cpp"
"typeindex": "cpp",
"*.tcc": "cpp",
"memory_resource": "cpp",
"numbers": "cpp",
"semaphore": "cpp",
"stop_token": "cpp",
"cfenv": "cpp"
}
}
+3 -3
View File
@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
project(
simdjson
# The version number is modified by tools/release.py
VERSION 3.7.0
VERSION 3.8.0
DESCRIPTION "Parsing gigabytes of JSON per second"
HOMEPAGE_URL "https://simdjson.org/"
LANGUAGES CXX C
@@ -20,8 +20,8 @@ string(
# ---- Options, variables ----
# These version numbers are modified by tools/release.py
set(SIMDJSON_LIB_VERSION "20.0.0" CACHE STRING "simdjson library version")
set(SIMDJSON_LIB_SOVERSION "20" CACHE STRING "simdjson library soversion")
set(SIMDJSON_LIB_VERSION "21.0.0" CACHE STRING "simdjson library version")
set(SIMDJSON_LIB_SOVERSION "21" CACHE STRING "simdjson library soversion")
option(SIMDJSON_BUILD_STATIC_LIB "Build simdjson_static library along with simdjson" OFF)
+1 -1
View File
@@ -38,7 +38,7 @@ PROJECT_NAME = simdjson
# could be handy for archiving the generated documentation or if some version
# control system is used.
PROJECT_NUMBER = "3.7.0"
PROJECT_NUMBER = "3.8.0"
# Using the PROJECT_BRIEF tag one can provide an optional one line description
# for a project that appears at the top of each page and should give viewer a
-4
View File
@@ -283,8 +283,6 @@ If your compiler does not default on C++11 support or better you may get failing
Note that the name of directory (`build`) is arbitrary, you can name it as you want (e.g., `buildgcc`) and you can have as many different such directories as you would like (one per configuration).
## Usage (CMake on 64-bit Windows using Visual Studio 2019 or better)
Recent versions of Visual Studio support CMake natively, [please refer to the Visual Studio documentation](https://learn.microsoft.com/en-us/cpp/build/cmake-projects-in-visual-studio?view=msvc-170).
@@ -309,13 +307,11 @@ Though having Visual Studio installed is necessary, one can build simdjson using
Furthermore, if you have installed LLVM clang on Windows, for example as a component of Visual Studio 2019, you can configure and build simdjson using LLVM clang on Windows using cmake:
- `mkdir build`
- `cd build`
- `cmake -T ClangCL ..`
- `cmake --build . --config Release`
## Various References
- [How to implement atoi using SIMD?](https://stackoverflow.com/questions/35127060/how-to-implement-atoi-using-simd)
+1
View File
@@ -169,6 +169,7 @@ We distinguish between "bindings" (which just wrap the C++ code) and a port to a
- [JSON-Simd](https://github.com/rawleyfowler/JSON-simd): Raku bindings.
- [JSON::SIMD](https://metacpan.org/pod/JSON::SIMD): Perl bindings; fully-featured JSON module that uses simdjson for decoding.
- [gemmaJSON](https://github.com/sainttttt/gemmaJSON): Nim JSON parser based on simdjson bindings.
- [simdjson-java](https://github.com/simdjson/simdjson-java): Java port.
About simdjson
--------------
-1
View File
@@ -993,7 +993,6 @@ struct AppleEvents {
// check permission
int force_ctrs = 0;
if (kpc_force_all_ctrs_get(&force_ctrs)) {
printf("Permission denied, xnu/kpc requires root privileges.\n");
return (worked = false);
}
int ret;
+4 -4
View File
@@ -22,7 +22,7 @@ endif()
# The bulk of our benchmarking and testing data has been moved simdjson/simdjson-data
import_dependency(simdjson-data simdjson/simdjson-data a5b13babe65c1bba7186b41b43d4cbdc20a5c470)
add_dependency(simdjson-data)
option(SIMDJSON_USE_BOOST_JSON "Try to include BOOST_JSON, this may break your binaries under some systems." OFF)
# This prevents variables declared with set() from unnecessarily escaping and
# should not be called more than once
function(competition_scope_)
@@ -36,8 +36,8 @@ function(competition_scope_)
#endif
int main() {}
]] USE_BOOST_JSON)
if(USE_BOOST_JSON)
]] SIMDJSON_FOUND_STRING_VIEW)
if(SIMDJSON_FOUND_STRING_VIEW AND SIMDJSON_USE_BOOST_JSON)
import_dependency(boostjson boostorg/json ee8d72d)
add_library(boostjson STATIC "${boostjson_SOURCE_DIR}/src/src.cpp")
target_compile_definitions(boostjson PUBLIC BOOST_JSON_STANDALONE)
@@ -134,7 +134,7 @@ int main() {}
add_library(competition-core INTERFACE)
target_link_libraries(competition-core INTERFACE nlohmann_json rapidjson sajson cjson jsmn yyjson)
if(USE_BOOST_JSON)
if(TARGET boostjson)
target_compile_definitions(boostjson INTERFACE HAS_BOOST_JSON)
target_link_libraries(competition-core INTERFACE boostjson)
endif()
+55 -1
View File
@@ -1910,8 +1910,10 @@ An `ondemand::number` instance may contain an integer value or a floating-point
Thus it is a dynamically typed number. Before accessing the value, you must determine the detected type:
* `number.get_number_type()` has value `number_type::signed_integer` if we have a integer in [-9223372036854775808,9223372036854775808). You can recover the value by the `get_int64()` method applied on the `ondemand::number` instance. When `number.get_number_type()` has value `number_type::signed_integer`, you also have that `number.is_int64()` is true. Calling `get_int64()` on the `ondemand::number` instance when `number.get_number_type()` is not `number_type::signed_integer` is unsafe. You may replace `get_int64()` by a cast to a `int64_t` value.
* `number.get_number_type()` has value `number_type::unsigned_integer` if we have a integer in [9223372036854775808,18446744073709551616). You can recover the value by the `get_uint64()` method applied on the `ondemand::number` instance. When `number.get_number_type()` has value `number_type::unsigned_integer`, you also have that `number.is_uint64()` is true. Calling `get_uint64()` on the `ondemand::number` instance when `number.get_number_type()` is not `number_type::unsigned_integer` is unsafe. You may replace `get_uint64()` by a cast to a `uint64_t` value.
* `number.get_number_type()` has value `number_type::unsigned_integer` if we have a integer in `[9223372036854775808,18446744073709551616)`. You can recover the value by the `get_uint64()` method applied on the `ondemand::number` instance. When `number.get_number_type()` has value `number_type::unsigned_integer`, you also have that `number.is_uint64()` is true. Calling `get_uint64()` on the `ondemand::number` instance when `number.get_number_type()` is not `number_type::unsigned_integer` is unsafe. You may replace `get_uint64()` by a cast to a `uint64_t` value.
* `number.get_number_type()` has value `number_type::floating_point_number` if we have and we have a floating-point (binary64) number. You can recover the value by the `get_double()` method applied on the `ondemand::number` instance. When `number.get_number_type()` has value `number_type::floating_point_number`, you also have that `number.is_double()` is true. Calling `get_double()` on the `ondemand::number` instance when `number.get_number_type()` is not `number_type::floating_point_number` is unsafe. You may replace `get_double()` by a cast to a `double` value.
* When the value is an integer outside of the valid ranges for a 64-bit integers, e.g., when it is smaller than -9223372036854775808 or larger than 18446744073709551615, then `number.get_number_type()` has value `number_type::big_integer`. If you try to parse
such a number of `get_number()`, you get the error `BIGINT_ERROR`. You can access the underlying string of digits with the function `raw_json_token()` which returns an `std::string_view` instance starting at the beginning of the digit. You can also call `get_double()` to get a floating-point approximation.
You must check the type before accessing the value: it is an error to call `get_int64()` when `number.get_number_type()` is not `number_type::signed_integer` and when `number.is_int64()` is false. You are responsible for this check as the user of the library.
@@ -1944,6 +1946,9 @@ Consider the following example:
std::cout << "float: " << double(num) << " ";
std::cout << "float: " << num.get_double() << std::endl;
break;
case ondemand::number_type::big_integer:
std::cout << "big-integer: " << val.raw_json_token() << std::endl;
break;
}
}
```
@@ -1959,6 +1964,55 @@ It will output:
9999999999999999999 negative: 0 is_integer: 1 large 64-bit integer: 9999999999999999999 large 64-bit integer: 9999999999999999999
```
In the following example, we have an array of integers that are outside the valid range of 64-bit signed or
unsigned integers. Calling `get_number_type()` on the values returns `ondemand::number_type::big_integer`.
You can try to represent these big integers as 64-bit floating-point numbers, though you typically lose
precision in the process (as illustrated in the example).
```C++
ondemand::parser parser;
padded_string docdata = R"([-9223372036854775809, 18446744073709551617, 99999999999999999999999 ])"_padded;
double dexpected[] = {-9223372036854775808.0, 18446744073709551616.0, 1e23};
ondemand::document doc = parser.iterate(docdata);
ondemand::array arr = doc.get_array();
for(ondemand::value val : arr) {
if(val.get_number_type() == ondemand::number_type::big_integer) {
std::cout << val.get_double() << std::endl;
// might print -9.22337e+18, 1.84467e+19, 1e+23
}
}
```
This program might print:
```
-9.22337e+18
1.84467e+19
1e+23
```
You may get access to the underlying string representing the big integer with
`raw_json_token()` and you may parse the resulting number strings using your own parser.
```c++
ondemand::parser parser;
padded_string docdata = R"([-9223372036854775809, 18446744073709551617, 99999999999999999999999 ])"_padded;
ondemand::document doc = parser.iterate(docdata);
ondemand::array arr = doc.get_array();
for(ondemand::value val : arr) {
// val.get_number_type() == ondemand::number_type::big_integer
if(val.get_number_type() == ondemand::number_type::big_integer) {
std::string_view token = val.raw_json_token();
// token = "-9223372036854775809", "18446744073709551617", "99999999999999999999999 "
std::cout << "'" << token << "'" << std::endl;
}
}
```
This code prints the following:
```
'-9223372036854775809'
'18446744073709551617'
'99999999999999999999999 '
```
Raw Strings From Keys
-----------
+1
View File
@@ -27,6 +27,7 @@ enum error_code {
F_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'f'
N_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'n'
NUMBER_ERROR, ///< Problem while parsing a number
BIGINT_ERROR, ///< The integer value exceeds 64 bits
UTF8_ERROR, ///< the input is not valid UTF-8
UNINITIALIZED, ///< unknown error, or uninitialized document
EMPTY, ///< no structural element found
+2 -1
View File
@@ -37,7 +37,8 @@ class dom_parser_implementation;
enum class number_type {
floating_point_number=1, /// a binary64 number
signed_integer, /// a signed integer that fits in a 64-bit word using two's complement
unsigned_integer /// a positive integer larger or equal to 1<<63
unsigned_integer, /// a positive integer larger or equal to 1<<63
big_integer /// a big integer that does not fit in a 64-bit word
};
} // namespace SIMDJSON_IMPLEMENTATION
+59 -10
View File
@@ -20,11 +20,13 @@ namespace numberparsing {
#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE)))
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE)))
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE)))
#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR)
#else
#define INVALID_NUMBER(SRC) (NUMBER_ERROR)
#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE))
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE))
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE))
#define BIGINT_NUMBER(SRC) (BIGINT_ERROR)
#endif
namespace {
@@ -355,6 +357,10 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) {
return true;
}
simdjson_inline bool is_digit(const uint8_t c) {
return static_cast<uint8_t>(c - '0') <= 9;
}
simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
// we continue with the fiction that we have an integer. If the
// floating point number is representable as x * 10^z for some integer
@@ -434,6 +440,23 @@ simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const s
return SUCCESS;
}
simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) {
const uint8_t *const srcend = src + max_length;
bool negative = (*src == '-'); // we can always read at least one character after the '-'
const uint8_t *p = src + uint8_t(negative);
if(p == srcend) { return false; }
if(*p == '0') {
++p;
if(p == srcend) { return true; }
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
return true;
}
while(p != srcend && is_digit(*p)) { ++p; }
if(p == srcend) { return true; }
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
return true;
}
simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) {
// It is possible that the integer had an overflow.
// We have to handle the case where we have 0.0000somenumber.
@@ -508,6 +531,18 @@ simdjson_inline error_code write_float(const uint8_t *const src, bool negative,
return SUCCESS;
}
// parse the number at src
// define JSON_TEST_NUMBERS for unit testing
//
// It is assumed that the number is followed by a structural ({,},],[) character
// or a white space character. If that is not the case (e.g., when the JSON
// document is made of a single number), then it is necessary to copy the
// content and append a space before calling this function.
//
// Our objective is accurate parsing (ULP of 0) at high speed.
template<typename W>
simdjson_inline error_code parse_number(const uint8_t *const src, W &writer);
// for performance analysis, it is sometimes useful to skip parsing
#ifdef SIMDJSON_SKIPNUMBERPARSING
@@ -586,11 +621,11 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
// The longest positive 64-bit number is 20 digits.
// We do it this way so we don't trigger this branch unless we must.
size_t longest_digit_count = negative ? 19 : 20;
if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); }
if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); }
if (digit_count == longest_digit_count) {
if (negative) {
// Anything negative above INT64_MAX+1 is invalid
if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src); }
if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); }
WRITE_INTEGER(~i+1, src, writer);
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
return SUCCESS;
@@ -1060,19 +1095,32 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
src += uint8_t(negative);
const uint8_t *p = src;
while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
size_t digit_count = size_t(p - src);
if ( p == src ) { return NUMBER_ERROR; }
if (jsoncharutils::is_structural_or_whitespace(*p)) {
static const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
// We have an integer.
if(simdjson_unlikely(digit_count > 20)) {
return number_type::big_integer;
}
// If the number is negative and valid, it must be a signed integer.
if(negative) { return number_type::signed_integer; }
if(negative) {
if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer;
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
return number_type::big_integer;
}
return number_type::signed_integer;
}
// Let us check if we have a big integer (>=2**64).
static const uint8_t * two_to_sixtyfour = reinterpret_cast<const uint8_t *>("18446744073709551616");
if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) {
return number_type::big_integer;
}
// The number is positive and smaller than 18446744073709551616 (or 2**64).
// We want values larger or equal to 9223372036854775808 to be unsigned
// integers, and the other values to be signed integers.
int digit_count = int(p - src);
if(digit_count >= 19) {
const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
return number_type::unsigned_integer;
}
if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) {
return number_type::unsigned_integer;
}
return number_type::signed_integer;
}
@@ -1250,6 +1298,7 @@ inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
case number_type::big_integer: out << "big integer"; break;
default: SIMDJSON_UNREACHABLE();
}
return out;
@@ -1258,4 +1307,4 @@ inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson
#endif // SIMDJSON_GENERIC_NUMBERPARSING_H
#endif // SIMDJSON_GENERIC_NUMBERPARSING_H
@@ -295,7 +295,7 @@ simdjson_inline simdjson_result<number> document::get_number() noexcept {
simdjson_inline simdjson_result<std::string_view> document::raw_json_token() noexcept {
auto _iter = get_root_value_iterator();
return std::string_view(reinterpret_cast<const char*>(_iter.peek_start()), _iter.peek_start_length());
return std::string_view(reinterpret_cast<const char*>(_iter.peek_start()), _iter.peek_root_length());
}
simdjson_inline simdjson_result<value> document::at_pointer(std::string_view json_pointer) noexcept {
@@ -535,6 +535,11 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> simdj
#if SIMDJSON_EXCEPTIONS
template <class T, typename std::enable_if<std::is_same<T, SIMDJSON_IMPLEMENTATION::ondemand::document>::value == false>::type>
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::operator T() noexcept(false) {
if (error()) { throw simdjson_error(error()); }
return first;
}
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::operator SIMDJSON_IMPLEMENTATION::ondemand::array() & noexcept(false) {
if (error()) { throw simdjson_error(error()); }
return first;
@@ -842,6 +847,11 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> simdj
return first.get_number();
}
#if SIMDJSON_EXCEPTIONS
template <class T, typename std::enable_if<std::is_same<T, SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::value == false>::type>
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator T() noexcept(false) {
if (error()) { throw simdjson_error(error()); }
return first;
}
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::operator SIMDJSON_IMPLEMENTATION::ondemand::array() & noexcept(false) {
if (error()) { throw simdjson_error(error()); }
return first;
+22 -9
View File
@@ -179,21 +179,23 @@ public:
* @returns INCORRECT_TYPE If the JSON value is not the given type.
*/
template<typename T> simdjson_inline simdjson_result<T> get() & noexcept {
// Unless the simdjson library provides an inline implementation, calling this method should
// Unless the simdjson library or the user provides an inline implementation, calling this method should
// immediately fail.
static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. "
"The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, "
"int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), "
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template.");
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."
" You may also add support for custom types, see our documentation.");
}
/** @overload template<typename T> simdjson_result<T> get() & noexcept */
template<typename T> simdjson_inline simdjson_result<T> get() && noexcept {
// Unless the simdjson library provides an inline implementation, calling this method should
// Unless the simdjson library or the user provides an inline implementation, calling this method should
// immediately fail.
static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. "
"The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, "
"int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), "
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template.");
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."
" You may also add support for custom types, see our documentation.");
}
/**
@@ -212,6 +214,15 @@ public:
template<typename T> simdjson_inline error_code get(T &out) && noexcept;
#if SIMDJSON_EXCEPTIONS
/**
* Cast this JSON value to an instance of type T. The programmer is responsible for
* providing an implementation of get<T> for the type T, if T is not one of the types
* supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.)
*
* See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types
*
* @returns An instance of type T
*/
template <class T>
explicit simdjson_inline operator T() noexcept(false);
/**
@@ -386,7 +397,7 @@ public:
* APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order.
*
* Use find_field() if you are sure fields will be in order (or are willing to treat it as if the
* field wasn't there when they aren't).
* field was not there when they are not in order).
*
* You must consume the fields on an object one at a time. A request for a new key
* invalidates previous field values: it makes them unsafe. E.g., the array
@@ -467,9 +478,11 @@ public:
* get_number().get_number_type().
*
* get_number_type() is number_type::unsigned_integer if we have
* an integer greater or equal to 9223372036854775808
* an integer greater or equal to 9223372036854775808 and no larger than 18446744073709551615.
* get_number_type() is number_type::signed_integer if we have an
* integer that is less than 9223372036854775808
* integer that is less than 9223372036854775808 and greater or equal to -9223372036854775808.
* get_number_type() is number_type::big_integer if we have an integer outside
* of those ranges (either larger than 18446744073709551615 or smaller than -9223372036854775808).
* Otherwise, get_number_type() has value number_type::floating_point_number
*
* This function requires processing the number string, but it is expected
@@ -782,7 +795,7 @@ public:
template<typename T> simdjson_inline error_code get(T &out) & noexcept;
template<typename T> simdjson_inline error_code get(T &out) && noexcept;
#if SIMDJSON_EXCEPTIONS
template <class T>
template <class T, typename std::enable_if<std::is_same<T, SIMDJSON_IMPLEMENTATION::ondemand::document>::value == false>::type>
explicit simdjson_inline operator T() noexcept(false);
simdjson_inline operator SIMDJSON_IMPLEMENTATION::ondemand::array() & noexcept(false);
simdjson_inline operator SIMDJSON_IMPLEMENTATION::ondemand::object() & noexcept(false);
@@ -853,7 +866,7 @@ public:
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> get_value() noexcept;
simdjson_inline simdjson_result<bool> is_null() noexcept;
#if SIMDJSON_EXCEPTIONS
template <class T>
template <class T, typename std::enable_if<std::is_same<T, SIMDJSON_IMPLEMENTATION::ondemand::document_reference>::value == false>::type>
explicit simdjson_inline operator T() noexcept(false);
simdjson_inline operator SIMDJSON_IMPLEMENTATION::ondemand::array() & noexcept(false);
simdjson_inline operator SIMDJSON_IMPLEMENTATION::ondemand::object() & noexcept(false);
@@ -286,6 +286,12 @@ simdjson_inline uint32_t json_iterator::peek_length(token_position position) con
#endif // SIMDJSON_CHECK_EOF
return token.peek_length(position);
}
simdjson_inline uint32_t json_iterator::peek_root_length(token_position position) const noexcept {
#if SIMDJSON_CHECK_EOF
assert_valid_position(position);
#endif // SIMDJSON_CHECK_EOF
return token.peek_root_length(position);
}
simdjson_inline token_position json_iterator::last_position() const noexcept {
// The following line fails under some compilers...
@@ -185,6 +185,14 @@ public:
* @param position The position of the token to retrieve.
*/
simdjson_inline uint32_t peek_length(token_position position) const noexcept;
/**
* Get the maximum length of the JSON text for the current root token.
*
* The length will include any whitespace at the end of the token.
*
* @param position The position of the token to retrieve.
*/
simdjson_inline uint32_t peek_root_length(token_position position) const noexcept;
/**
* Get the JSON text for the last token in the document.
*
@@ -48,7 +48,6 @@ simdjson_inline number::operator uint64_t() const noexcept {
return get_uint64();
}
simdjson_inline bool number::is_int64() const noexcept {
return get_number_type() == number_type::signed_integer;
}
+1 -1
View File
@@ -77,7 +77,7 @@ public:
* APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order.
*
* Use find_field() if you are sure fields will be in order (or are willing to treat it as if the
* field wasn't there when they aren't).
* field was not there when they are not in order).
*
* If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful
* that only one field is returned.
@@ -37,6 +37,11 @@ simdjson_inline uint32_t token_iterator::peek_length(token_position position) co
return *(position+1) - *position;
}
simdjson_inline uint32_t token_iterator::peek_root_length(token_position position) const noexcept {
return *(position+2) - *(position) > *(position+1) - *(position) ?
*(position+1) - *(position)
: *(position+2) - *(position);
}
simdjson_inline const uint8_t *token_iterator::peek(int32_t delta) const noexcept {
return &buf[*(_position+delta)];
}
@@ -77,7 +77,14 @@ public:
* @param position The position of the token.
*/
simdjson_inline uint32_t peek_length(token_position position) const noexcept;
/**
* Get the maximum length of the JSON text for a root token.
*
* The length will include any whitespace at the end of the token.
*
* @param position The position of the token (start of the document).
*/
simdjson_inline uint32_t peek_root_length(token_position position) const noexcept;
/**
* Return the current index.
*/
+16 -9
View File
@@ -36,12 +36,13 @@ public:
* @returns INCORRECT_TYPE If the JSON value is not the given type.
*/
template<typename T> simdjson_inline simdjson_result<T> get() noexcept {
// Unless the simdjson library provides an inline implementation, calling this method should
// Unless the simdjson library or the user provides an inline implementation, calling this method should
// immediately fail.
static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. "
"The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, "
"int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), "
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template.");
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."
" You may also add support for custom types, see our documentation.");
}
/**
@@ -196,7 +197,10 @@ public:
#if SIMDJSON_EXCEPTIONS
/**
* Cast this JSON value to an instance of type T. The programmer is responsible for
* providing an implementation of get<T> for the type T.
* providing an implementation of get<T> for the type T, if T is not one of the types
* supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.).
*
* See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types
*
* @returns An instance of type T
*/
@@ -365,7 +369,7 @@ public:
* that only one field is returned.
*
* Use find_field() if you are sure fields will be in order (or are willing to treat it as if the
* field wasn't there when they aren't).
* field as not there when they are not in order).
*
* @param key The key to look up.
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
@@ -440,10 +444,12 @@ public:
* get_number().get_number_type().
*
* get_number_type() is number_type::unsigned_integer if we have
* an integer greater or equal to 9223372036854775808
* an integer greater or equal to 9223372036854775808.
* get_number_type() is number_type::signed_integer if we have an
* integer that is less than 9223372036854775808
* Otherwise, get_number_type() has value number_type::floating_point_number
* integer that is less than 9223372036854775808.
* get_number_type() is number_type::big_integer for integers that do not fit in 64 bits,
* in which case the digit_count is set to the length of the big integer string.
* Otherwise, get_number_type() has value number_type::floating_point_number.
*
* This function requires processing the number string, but it is expected
* to be faster than get_number().get_number_type() because it is does not
@@ -470,6 +476,8 @@ public:
* You can recover the value by calling number.get_uint64() and you
* have that number.is_uint64() is true.
*
* For integers that do not fit in 64 bits, the function returns BIGINT_ERROR error code.
*
* Otherwise, number.get_number_type() has value number_type::floating_point_number
* and we have a binary64 number.
* You can recover the value by calling number.get_double() and you
@@ -485,7 +493,6 @@ public:
*/
simdjson_warn_unused simdjson_inline simdjson_result<number> get_number() noexcept;
/**
* Get the raw JSON for this token.
*
@@ -729,7 +736,7 @@ public:
* APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order.
*
* Use find_field() if you are sure fields will be in order (or are willing to treat it as if the
* field wasn't there when they aren't).
* field as not there when they are not in order).
*
* @param key The key to look up.
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
@@ -137,7 +137,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::find_
} else if (!is_open()) {
#if SIMDJSON_DEVELOPMENT_CHECKS
// If we're past the end of the object, we're being iterated out of order.
// Note: this isn't perfect detection. It's possible the user is inside some other object; if so,
// Note: this is not perfect detection. It's possible the user is inside some other object; if so,
// this object iterator will blithely scan that object for fields.
if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; }
#endif
@@ -249,7 +249,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::find_
#if SIMDJSON_DEVELOPMENT_CHECKS
// If we're past the end of the object, we're being iterated out of order.
// Note: this isn't perfect detection. It's possible the user is inside some other object; if so,
// Note: this is not perfect detection. It's possible the user is inside some other object; if so,
// this object iterator will blithely scan that object for fields.
if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; }
#endif
@@ -591,7 +591,7 @@ simdjson_inline simdjson_result<number> value_iterator::get_number() noexcept {
}
simdjson_inline simdjson_result<bool> value_iterator::is_root_integer(bool check_trailing) noexcept {
auto max_len = peek_start_length();
auto max_len = peek_root_length();
auto json = peek_root_scalar("is_root_integer");
uint8_t tmpbuf[20+1+1]{}; // <20 digits> is the longest possible unsigned integer
tmpbuf[20+1] = '\0'; // make sure that buffer is always null terminated.
@@ -607,7 +607,7 @@ simdjson_inline simdjson_result<bool> value_iterator::is_root_integer(bool check
}
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::number_type> value_iterator::get_root_number_type(bool check_trailing) noexcept {
auto max_len = peek_start_length();
auto max_len = peek_root_length();
auto json = peek_root_scalar("number");
// Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/,
// 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest
@@ -615,7 +615,12 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::number_type> value_iter
uint8_t tmpbuf[1074+8+1+1];
tmpbuf[1074+8+1] = '\0'; // make sure that buffer is always null terminated.
if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074+8+1)) {
logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters");
if(numberparsing::check_if_integer(json, max_len)) {
if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; }
logger::log_error(*_json_iter, start_position(), depth(), "Found big integer");
return number_type::big_integer;
}
logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer");
return NUMBER_ERROR;
}
auto answer = numberparsing::get_number_type(tmpbuf);
@@ -623,15 +628,21 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::number_type> value_iter
return answer;
}
simdjson_inline simdjson_result<number> value_iterator::get_root_number(bool check_trailing) noexcept {
auto max_len = peek_start_length();
auto max_len = peek_root_length();
auto json = peek_root_scalar("number");
// Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/,
// 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest
// number: -0.<fraction>e-308.
// NOTE: the current approach doesn't work for very big integer numbers containing more than 1074 digits.
uint8_t tmpbuf[1074+8+1+1];
tmpbuf[1074+8+1] = '\0'; // make sure that buffer is always null terminated.
if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074+8+1)) {
logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters");
if(numberparsing::check_if_integer(json, max_len)) {
if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; }
logger::log_error(*_json_iter, start_position(), depth(), "Found big integer");
return BIGINT_ERROR;
}
logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer");
return NUMBER_ERROR;
}
number num;
@@ -663,7 +674,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> value_iter
return raw_json_string(json+1);
}
simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> value_iterator::get_root_uint64(bool check_trailing) noexcept {
auto max_len = peek_start_length();
auto max_len = peek_root_length();
auto json = peek_root_scalar("uint64");
uint8_t tmpbuf[20+1+1]{}; // <20 digits> is the longest possible unsigned integer
tmpbuf[20+1] = '\0'; // make sure that buffer is always null terminated.
@@ -679,7 +690,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> value_iterator::g
return result;
}
simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> value_iterator::get_root_uint64_in_string(bool check_trailing) noexcept {
auto max_len = peek_start_length();
auto max_len = peek_root_length();
auto json = peek_root_scalar("uint64");
uint8_t tmpbuf[20+1+1]{}; // <20 digits> is the longest possible unsigned integer
tmpbuf[20+1] = '\0'; // make sure that buffer is always null terminated.
@@ -695,7 +706,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<uint64_t> value_iterator::g
return result;
}
simdjson_warn_unused simdjson_inline simdjson_result<int64_t> value_iterator::get_root_int64(bool check_trailing) noexcept {
auto max_len = peek_start_length();
auto max_len = peek_root_length();
auto json = peek_root_scalar("int64");
uint8_t tmpbuf[20+1+1]; // -<19 digits> is the longest possible integer
tmpbuf[20+1] = '\0'; // make sure that buffer is always null terminated.
@@ -712,7 +723,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<int64_t> value_iterator::ge
return result;
}
simdjson_warn_unused simdjson_inline simdjson_result<int64_t> value_iterator::get_root_int64_in_string(bool check_trailing) noexcept {
auto max_len = peek_start_length();
auto max_len = peek_root_length();
auto json = peek_root_scalar("int64");
uint8_t tmpbuf[20+1+1]; // -<19 digits> is the longest possible integer
tmpbuf[20+1] = '\0'; // make sure that buffer is always null terminated.
@@ -729,7 +740,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<int64_t> value_iterator::ge
return result;
}
simdjson_warn_unused simdjson_inline simdjson_result<double> value_iterator::get_root_double(bool check_trailing) noexcept {
auto max_len = peek_start_length();
auto max_len = peek_root_length();
auto json = peek_root_scalar("double");
// Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/,
// 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest
@@ -749,7 +760,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<double> value_iterator::get
}
simdjson_warn_unused simdjson_inline simdjson_result<double> value_iterator::get_root_double_in_string(bool check_trailing) noexcept {
auto max_len = peek_start_length();
auto max_len = peek_root_length();
auto json = peek_root_scalar("double");
// Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/,
// 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest
@@ -768,7 +779,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<double> value_iterator::get
return result;
}
simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::get_root_bool(bool check_trailing) noexcept {
auto max_len = peek_start_length();
auto max_len = peek_root_length();
auto json = peek_root_scalar("bool");
uint8_t tmpbuf[5+1+1]; // +1 for null termination
tmpbuf[5+1] = '\0'; // make sure that buffer is always null terminated.
@@ -781,7 +792,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<bool> value_iterator::get_r
return result;
}
simdjson_inline simdjson_result<bool> value_iterator::is_root_null(bool check_trailing) noexcept {
auto max_len = peek_start_length();
auto max_len = peek_root_length();
auto json = peek_root_scalar("null");
bool result = (max_len >= 4 && !atomparsing::str4ncmp(json, "null") &&
(max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[4])));
@@ -853,6 +864,9 @@ simdjson_inline const uint8_t *value_iterator::peek_start() const noexcept {
simdjson_inline uint32_t value_iterator::peek_start_length() const noexcept {
return _json_iter->peek_length(start_position());
}
simdjson_inline uint32_t value_iterator::peek_root_length() const noexcept {
return _json_iter->peek_root_length(start_position());
}
simdjson_inline const uint8_t *value_iterator::peek_scalar(const char *type) noexcept {
logger::log_value(*_json_iter, start_position(), depth(), type);
@@ -375,6 +375,7 @@ protected:
simdjson_inline simdjson_result<bool> parse_bool(const uint8_t *json) const noexcept;
simdjson_inline const uint8_t *peek_start() const noexcept;
simdjson_inline uint32_t peek_start_length() const noexcept;
simdjson_inline uint32_t peek_root_length() const noexcept;
/**
* The general idea of the advance_... methods and the peek_* methods
+2
View File
@@ -38,6 +38,8 @@
#define SIMDJSON_IS_ARM64 1
#elif defined(__riscv) && __riscv_xlen == 64
#define SIMDJSON_IS_RISCV64 1
#elif defined(__loongarch_lp64)
#define SIMDJSON_IS_LOONGARCH64 1
#elif defined(__PPC64__) || defined(_M_PPC64)
#if defined(__ALTIVEC__)
#define SIMDJSON_IS_PPC64_VMX 1
+2 -2
View File
@@ -4,7 +4,7 @@
#define SIMDJSON_SIMDJSON_VERSION_H
/** The version of simdjson being used (major.minor.revision) */
#define SIMDJSON_VERSION "3.7.0"
#define SIMDJSON_VERSION "3.8.0"
namespace simdjson {
enum {
@@ -15,7 +15,7 @@ enum {
/**
* The minor version (major.MINOR.revision) of simdjson being used.
*/
SIMDJSON_VERSION_MINOR = 7,
SIMDJSON_VERSION_MINOR = 8,
/**
* The revision (major.minor.REVISION) of simdjson being used.
*/
+389 -85
View File
@@ -1,4 +1,4 @@
/* auto-generated on 2024-02-18 11:50:12 -0500. Do not edit! */
/* auto-generated on 2024-03-10 13:24:07 -0400. Do not edit! */
/* including simdjson.cpp: */
/* begin file simdjson.cpp */
#define SIMDJSON_SRC_SIMDJSON_CPP
@@ -111,6 +111,8 @@
#define SIMDJSON_IS_ARM64 1
#elif defined(__riscv) && __riscv_xlen == 64
#define SIMDJSON_IS_RISCV64 1
#elif defined(__loongarch_lp64)
#define SIMDJSON_IS_LOONGARCH64 1
#elif defined(__PPC64__) || defined(_M_PPC64)
#if defined(__ALTIVEC__)
#define SIMDJSON_IS_PPC64_VMX 1
@@ -2345,6 +2347,7 @@ enum error_code {
F_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'f'
N_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'n'
NUMBER_ERROR, ///< Problem while parsing a number
BIGINT_ERROR, ///< The integer value exceeds 64 bits
UTF8_ERROR, ///< the input is not valid UTF-8
UNINITIALIZED, ///< unknown error, or uninitialized document
EMPTY, ///< no structural element found
@@ -4493,6 +4496,7 @@ namespace internal {
{ F_ATOM_ERROR, "F_ATOM_ERROR: Problem while parsing an atom starting with the letter 'f'" },
{ N_ATOM_ERROR, "N_ATOM_ERROR: Problem while parsing an atom starting with the letter 'n'" },
{ NUMBER_ERROR, "NUMBER_ERROR: Problem while parsing a number" },
{ BIGINT_ERROR, "BIGINT_ERROR: Big integer value that cannot be represented using 64 bits" },
{ UTF8_ERROR, "UTF8_ERROR: The input is not valid UTF-8" },
{ UNINITIALIZED, "UNINITIALIZED: Uninitialized" },
{ EMPTY, "EMPTY: no JSON found" },
@@ -8213,7 +8217,8 @@ class dom_parser_implementation;
enum class number_type {
floating_point_number=1, /// a binary64 number
signed_integer, /// a signed integer that fits in a 64-bit word using two's complement
unsigned_integer /// a positive integer larger or equal to 1<<63
unsigned_integer, /// a positive integer larger or equal to 1<<63
big_integer /// a big integer that does not fit in a 64-bit word
};
} // namespace arm64
@@ -8661,11 +8666,13 @@ namespace numberparsing {
#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE)))
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE)))
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE)))
#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR)
#else
#define INVALID_NUMBER(SRC) (NUMBER_ERROR)
#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE))
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE))
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE))
#define BIGINT_NUMBER(SRC) (BIGINT_ERROR)
#endif
namespace {
@@ -8996,6 +9003,10 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) {
return true;
}
simdjson_inline bool is_digit(const uint8_t c) {
return static_cast<uint8_t>(c - '0') <= 9;
}
simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
// we continue with the fiction that we have an integer. If the
// floating point number is representable as x * 10^z for some integer
@@ -9075,6 +9086,23 @@ simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const s
return SUCCESS;
}
simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) {
const uint8_t *const srcend = src + max_length;
bool negative = (*src == '-'); // we can always read at least one character after the '-'
const uint8_t *p = src + uint8_t(negative);
if(p == srcend) { return false; }
if(*p == '0') {
++p;
if(p == srcend) { return true; }
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
return true;
}
while(p != srcend && is_digit(*p)) { ++p; }
if(p == srcend) { return true; }
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
return true;
}
simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) {
// It is possible that the integer had an overflow.
// We have to handle the case where we have 0.0000somenumber.
@@ -9149,6 +9177,18 @@ simdjson_inline error_code write_float(const uint8_t *const src, bool negative,
return SUCCESS;
}
// parse the number at src
// define JSON_TEST_NUMBERS for unit testing
//
// It is assumed that the number is followed by a structural ({,},],[) character
// or a white space character. If that is not the case (e.g., when the JSON
// document is made of a single number), then it is necessary to copy the
// content and append a space before calling this function.
//
// Our objective is accurate parsing (ULP of 0) at high speed.
template<typename W>
simdjson_inline error_code parse_number(const uint8_t *const src, W &writer);
// for performance analysis, it is sometimes useful to skip parsing
#ifdef SIMDJSON_SKIPNUMBERPARSING
@@ -9227,11 +9267,11 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
// The longest positive 64-bit number is 20 digits.
// We do it this way so we don't trigger this branch unless we must.
size_t longest_digit_count = negative ? 19 : 20;
if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); }
if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); }
if (digit_count == longest_digit_count) {
if (negative) {
// Anything negative above INT64_MAX+1 is invalid
if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src); }
if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); }
WRITE_INTEGER(~i+1, src, writer);
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
return SUCCESS;
@@ -9701,19 +9741,32 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
src += uint8_t(negative);
const uint8_t *p = src;
while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
size_t digit_count = size_t(p - src);
if ( p == src ) { return NUMBER_ERROR; }
if (jsoncharutils::is_structural_or_whitespace(*p)) {
static const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
// We have an integer.
if(simdjson_unlikely(digit_count > 20)) {
return number_type::big_integer;
}
// If the number is negative and valid, it must be a signed integer.
if(negative) { return number_type::signed_integer; }
if(negative) {
if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer;
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
return number_type::big_integer;
}
return number_type::signed_integer;
}
// Let us check if we have a big integer (>=2**64).
static const uint8_t * two_to_sixtyfour = reinterpret_cast<const uint8_t *>("18446744073709551616");
if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) {
return number_type::big_integer;
}
// The number is positive and smaller than 18446744073709551616 (or 2**64).
// We want values larger or equal to 9223372036854775808 to be unsigned
// integers, and the other values to be signed integers.
int digit_count = int(p - src);
if(digit_count >= 19) {
const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
return number_type::unsigned_integer;
}
if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) {
return number_type::unsigned_integer;
}
return number_type::signed_integer;
}
@@ -9891,6 +9944,7 @@ inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
case number_type::big_integer: out << "big integer"; break;
default: SIMDJSON_UNREACHABLE();
}
return out;
@@ -13019,8 +13073,8 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
// Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD)
constexpr uint32_t substitution_code_point = 0xfffd;
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
// conversion isn't valid; we defer the check for this to inside the
// multilingual plane check
// conversion is not valid; we defer the check for this to inside the
// multilingual plane check.
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
*src_ptr += 6;
@@ -13077,8 +13131,8 @@ simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr,
// It is not ideal that this function is nearly identical to handle_unicode_codepoint.
//
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
// conversion isn't valid; we defer the check for this to inside the
// multilingual plane check
// conversion is not valid; we defer the check for this to inside the
// multilingual plane check.
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
*src_ptr += 6;
// If we found a high surrogate, we must
@@ -14501,7 +14555,8 @@ class dom_parser_implementation;
enum class number_type {
floating_point_number=1, /// a binary64 number
signed_integer, /// a signed integer that fits in a 64-bit word using two's complement
unsigned_integer /// a positive integer larger or equal to 1<<63
unsigned_integer, /// a positive integer larger or equal to 1<<63
big_integer /// a big integer that does not fit in a 64-bit word
};
} // namespace haswell
@@ -14949,11 +15004,13 @@ namespace numberparsing {
#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE)))
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE)))
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE)))
#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR)
#else
#define INVALID_NUMBER(SRC) (NUMBER_ERROR)
#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE))
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE))
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE))
#define BIGINT_NUMBER(SRC) (BIGINT_ERROR)
#endif
namespace {
@@ -15284,6 +15341,10 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) {
return true;
}
simdjson_inline bool is_digit(const uint8_t c) {
return static_cast<uint8_t>(c - '0') <= 9;
}
simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
// we continue with the fiction that we have an integer. If the
// floating point number is representable as x * 10^z for some integer
@@ -15363,6 +15424,23 @@ simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const s
return SUCCESS;
}
simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) {
const uint8_t *const srcend = src + max_length;
bool negative = (*src == '-'); // we can always read at least one character after the '-'
const uint8_t *p = src + uint8_t(negative);
if(p == srcend) { return false; }
if(*p == '0') {
++p;
if(p == srcend) { return true; }
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
return true;
}
while(p != srcend && is_digit(*p)) { ++p; }
if(p == srcend) { return true; }
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
return true;
}
simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) {
// It is possible that the integer had an overflow.
// We have to handle the case where we have 0.0000somenumber.
@@ -15437,6 +15515,18 @@ simdjson_inline error_code write_float(const uint8_t *const src, bool negative,
return SUCCESS;
}
// parse the number at src
// define JSON_TEST_NUMBERS for unit testing
//
// It is assumed that the number is followed by a structural ({,},],[) character
// or a white space character. If that is not the case (e.g., when the JSON
// document is made of a single number), then it is necessary to copy the
// content and append a space before calling this function.
//
// Our objective is accurate parsing (ULP of 0) at high speed.
template<typename W>
simdjson_inline error_code parse_number(const uint8_t *const src, W &writer);
// for performance analysis, it is sometimes useful to skip parsing
#ifdef SIMDJSON_SKIPNUMBERPARSING
@@ -15515,11 +15605,11 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
// The longest positive 64-bit number is 20 digits.
// We do it this way so we don't trigger this branch unless we must.
size_t longest_digit_count = negative ? 19 : 20;
if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); }
if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); }
if (digit_count == longest_digit_count) {
if (negative) {
// Anything negative above INT64_MAX+1 is invalid
if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src); }
if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); }
WRITE_INTEGER(~i+1, src, writer);
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
return SUCCESS;
@@ -15989,19 +16079,32 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
src += uint8_t(negative);
const uint8_t *p = src;
while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
size_t digit_count = size_t(p - src);
if ( p == src ) { return NUMBER_ERROR; }
if (jsoncharutils::is_structural_or_whitespace(*p)) {
static const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
// We have an integer.
if(simdjson_unlikely(digit_count > 20)) {
return number_type::big_integer;
}
// If the number is negative and valid, it must be a signed integer.
if(negative) { return number_type::signed_integer; }
if(negative) {
if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer;
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
return number_type::big_integer;
}
return number_type::signed_integer;
}
// Let us check if we have a big integer (>=2**64).
static const uint8_t * two_to_sixtyfour = reinterpret_cast<const uint8_t *>("18446744073709551616");
if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) {
return number_type::big_integer;
}
// The number is positive and smaller than 18446744073709551616 (or 2**64).
// We want values larger or equal to 9223372036854775808 to be unsigned
// integers, and the other values to be signed integers.
int digit_count = int(p - src);
if(digit_count >= 19) {
const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
return number_type::unsigned_integer;
}
if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) {
return number_type::unsigned_integer;
}
return number_type::signed_integer;
}
@@ -16179,6 +16282,7 @@ inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
case number_type::big_integer: out << "big integer"; break;
default: SIMDJSON_UNREACHABLE();
}
return out;
@@ -19184,8 +19288,8 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
// Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD)
constexpr uint32_t substitution_code_point = 0xfffd;
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
// conversion isn't valid; we defer the check for this to inside the
// multilingual plane check
// conversion is not valid; we defer the check for this to inside the
// multilingual plane check.
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
*src_ptr += 6;
@@ -19242,8 +19346,8 @@ simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr,
// It is not ideal that this function is nearly identical to handle_unicode_codepoint.
//
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
// conversion isn't valid; we defer the check for this to inside the
// multilingual plane check
// conversion is not valid; we defer the check for this to inside the
// multilingual plane check.
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
*src_ptr += 6;
// If we found a high surrogate, we must
@@ -20661,7 +20765,8 @@ class dom_parser_implementation;
enum class number_type {
floating_point_number=1, /// a binary64 number
signed_integer, /// a signed integer that fits in a 64-bit word using two's complement
unsigned_integer /// a positive integer larger or equal to 1<<63
unsigned_integer, /// a positive integer larger or equal to 1<<63
big_integer /// a big integer that does not fit in a 64-bit word
};
} // namespace icelake
@@ -21109,11 +21214,13 @@ namespace numberparsing {
#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE)))
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE)))
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE)))
#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR)
#else
#define INVALID_NUMBER(SRC) (NUMBER_ERROR)
#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE))
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE))
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE))
#define BIGINT_NUMBER(SRC) (BIGINT_ERROR)
#endif
namespace {
@@ -21444,6 +21551,10 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) {
return true;
}
simdjson_inline bool is_digit(const uint8_t c) {
return static_cast<uint8_t>(c - '0') <= 9;
}
simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
// we continue with the fiction that we have an integer. If the
// floating point number is representable as x * 10^z for some integer
@@ -21523,6 +21634,23 @@ simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const s
return SUCCESS;
}
simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) {
const uint8_t *const srcend = src + max_length;
bool negative = (*src == '-'); // we can always read at least one character after the '-'
const uint8_t *p = src + uint8_t(negative);
if(p == srcend) { return false; }
if(*p == '0') {
++p;
if(p == srcend) { return true; }
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
return true;
}
while(p != srcend && is_digit(*p)) { ++p; }
if(p == srcend) { return true; }
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
return true;
}
simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) {
// It is possible that the integer had an overflow.
// We have to handle the case where we have 0.0000somenumber.
@@ -21597,6 +21725,18 @@ simdjson_inline error_code write_float(const uint8_t *const src, bool negative,
return SUCCESS;
}
// parse the number at src
// define JSON_TEST_NUMBERS for unit testing
//
// It is assumed that the number is followed by a structural ({,},],[) character
// or a white space character. If that is not the case (e.g., when the JSON
// document is made of a single number), then it is necessary to copy the
// content and append a space before calling this function.
//
// Our objective is accurate parsing (ULP of 0) at high speed.
template<typename W>
simdjson_inline error_code parse_number(const uint8_t *const src, W &writer);
// for performance analysis, it is sometimes useful to skip parsing
#ifdef SIMDJSON_SKIPNUMBERPARSING
@@ -21675,11 +21815,11 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
// The longest positive 64-bit number is 20 digits.
// We do it this way so we don't trigger this branch unless we must.
size_t longest_digit_count = negative ? 19 : 20;
if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); }
if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); }
if (digit_count == longest_digit_count) {
if (negative) {
// Anything negative above INT64_MAX+1 is invalid
if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src); }
if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); }
WRITE_INTEGER(~i+1, src, writer);
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
return SUCCESS;
@@ -22149,19 +22289,32 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
src += uint8_t(negative);
const uint8_t *p = src;
while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
size_t digit_count = size_t(p - src);
if ( p == src ) { return NUMBER_ERROR; }
if (jsoncharutils::is_structural_or_whitespace(*p)) {
static const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
// We have an integer.
if(simdjson_unlikely(digit_count > 20)) {
return number_type::big_integer;
}
// If the number is negative and valid, it must be a signed integer.
if(negative) { return number_type::signed_integer; }
if(negative) {
if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer;
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
return number_type::big_integer;
}
return number_type::signed_integer;
}
// Let us check if we have a big integer (>=2**64).
static const uint8_t * two_to_sixtyfour = reinterpret_cast<const uint8_t *>("18446744073709551616");
if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) {
return number_type::big_integer;
}
// The number is positive and smaller than 18446744073709551616 (or 2**64).
// We want values larger or equal to 9223372036854775808 to be unsigned
// integers, and the other values to be signed integers.
int digit_count = int(p - src);
if(digit_count >= 19) {
const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
return number_type::unsigned_integer;
}
if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) {
return number_type::unsigned_integer;
}
return number_type::signed_integer;
}
@@ -22339,6 +22492,7 @@ inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
case number_type::big_integer: out << "big integer"; break;
default: SIMDJSON_UNREACHABLE();
}
return out;
@@ -25342,8 +25496,8 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
// Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD)
constexpr uint32_t substitution_code_point = 0xfffd;
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
// conversion isn't valid; we defer the check for this to inside the
// multilingual plane check
// conversion is not valid; we defer the check for this to inside the
// multilingual plane check.
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
*src_ptr += 6;
@@ -25400,8 +25554,8 @@ simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr,
// It is not ideal that this function is nearly identical to handle_unicode_codepoint.
//
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
// conversion isn't valid; we defer the check for this to inside the
// multilingual plane check
// conversion is not valid; we defer the check for this to inside the
// multilingual plane check.
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
*src_ptr += 6;
// If we found a high surrogate, we must
@@ -26977,7 +27131,8 @@ class dom_parser_implementation;
enum class number_type {
floating_point_number=1, /// a binary64 number
signed_integer, /// a signed integer that fits in a 64-bit word using two's complement
unsigned_integer /// a positive integer larger or equal to 1<<63
unsigned_integer, /// a positive integer larger or equal to 1<<63
big_integer /// a big integer that does not fit in a 64-bit word
};
} // namespace ppc64
@@ -27425,11 +27580,13 @@ namespace numberparsing {
#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE)))
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE)))
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE)))
#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR)
#else
#define INVALID_NUMBER(SRC) (NUMBER_ERROR)
#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE))
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE))
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE))
#define BIGINT_NUMBER(SRC) (BIGINT_ERROR)
#endif
namespace {
@@ -27760,6 +27917,10 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) {
return true;
}
simdjson_inline bool is_digit(const uint8_t c) {
return static_cast<uint8_t>(c - '0') <= 9;
}
simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
// we continue with the fiction that we have an integer. If the
// floating point number is representable as x * 10^z for some integer
@@ -27839,6 +28000,23 @@ simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const s
return SUCCESS;
}
simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) {
const uint8_t *const srcend = src + max_length;
bool negative = (*src == '-'); // we can always read at least one character after the '-'
const uint8_t *p = src + uint8_t(negative);
if(p == srcend) { return false; }
if(*p == '0') {
++p;
if(p == srcend) { return true; }
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
return true;
}
while(p != srcend && is_digit(*p)) { ++p; }
if(p == srcend) { return true; }
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
return true;
}
simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) {
// It is possible that the integer had an overflow.
// We have to handle the case where we have 0.0000somenumber.
@@ -27913,6 +28091,18 @@ simdjson_inline error_code write_float(const uint8_t *const src, bool negative,
return SUCCESS;
}
// parse the number at src
// define JSON_TEST_NUMBERS for unit testing
//
// It is assumed that the number is followed by a structural ({,},],[) character
// or a white space character. If that is not the case (e.g., when the JSON
// document is made of a single number), then it is necessary to copy the
// content and append a space before calling this function.
//
// Our objective is accurate parsing (ULP of 0) at high speed.
template<typename W>
simdjson_inline error_code parse_number(const uint8_t *const src, W &writer);
// for performance analysis, it is sometimes useful to skip parsing
#ifdef SIMDJSON_SKIPNUMBERPARSING
@@ -27991,11 +28181,11 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
// The longest positive 64-bit number is 20 digits.
// We do it this way so we don't trigger this branch unless we must.
size_t longest_digit_count = negative ? 19 : 20;
if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); }
if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); }
if (digit_count == longest_digit_count) {
if (negative) {
// Anything negative above INT64_MAX+1 is invalid
if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src); }
if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); }
WRITE_INTEGER(~i+1, src, writer);
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
return SUCCESS;
@@ -28465,19 +28655,32 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
src += uint8_t(negative);
const uint8_t *p = src;
while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
size_t digit_count = size_t(p - src);
if ( p == src ) { return NUMBER_ERROR; }
if (jsoncharutils::is_structural_or_whitespace(*p)) {
static const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
// We have an integer.
if(simdjson_unlikely(digit_count > 20)) {
return number_type::big_integer;
}
// If the number is negative and valid, it must be a signed integer.
if(negative) { return number_type::signed_integer; }
if(negative) {
if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer;
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
return number_type::big_integer;
}
return number_type::signed_integer;
}
// Let us check if we have a big integer (>=2**64).
static const uint8_t * two_to_sixtyfour = reinterpret_cast<const uint8_t *>("18446744073709551616");
if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) {
return number_type::big_integer;
}
// The number is positive and smaller than 18446744073709551616 (or 2**64).
// We want values larger or equal to 9223372036854775808 to be unsigned
// integers, and the other values to be signed integers.
int digit_count = int(p - src);
if(digit_count >= 19) {
const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
return number_type::unsigned_integer;
}
if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) {
return number_type::unsigned_integer;
}
return number_type::signed_integer;
}
@@ -28655,6 +28858,7 @@ inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
case number_type::big_integer: out << "big integer"; break;
default: SIMDJSON_UNREACHABLE();
}
return out;
@@ -31771,8 +31975,8 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
// Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD)
constexpr uint32_t substitution_code_point = 0xfffd;
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
// conversion isn't valid; we defer the check for this to inside the
// multilingual plane check
// conversion is not valid; we defer the check for this to inside the
// multilingual plane check.
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
*src_ptr += 6;
@@ -31829,8 +32033,8 @@ simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr,
// It is not ideal that this function is nearly identical to handle_unicode_codepoint.
//
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
// conversion isn't valid; we defer the check for this to inside the
// multilingual plane check
// conversion is not valid; we defer the check for this to inside the
// multilingual plane check.
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
*src_ptr += 6;
// If we found a high surrogate, we must
@@ -33659,7 +33863,8 @@ class dom_parser_implementation;
enum class number_type {
floating_point_number=1, /// a binary64 number
signed_integer, /// a signed integer that fits in a 64-bit word using two's complement
unsigned_integer /// a positive integer larger or equal to 1<<63
unsigned_integer, /// a positive integer larger or equal to 1<<63
big_integer /// a big integer that does not fit in a 64-bit word
};
} // namespace westmere
@@ -34107,11 +34312,13 @@ namespace numberparsing {
#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE)))
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE)))
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE)))
#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR)
#else
#define INVALID_NUMBER(SRC) (NUMBER_ERROR)
#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE))
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE))
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE))
#define BIGINT_NUMBER(SRC) (BIGINT_ERROR)
#endif
namespace {
@@ -34442,6 +34649,10 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) {
return true;
}
simdjson_inline bool is_digit(const uint8_t c) {
return static_cast<uint8_t>(c - '0') <= 9;
}
simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
// we continue with the fiction that we have an integer. If the
// floating point number is representable as x * 10^z for some integer
@@ -34521,6 +34732,23 @@ simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const s
return SUCCESS;
}
simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) {
const uint8_t *const srcend = src + max_length;
bool negative = (*src == '-'); // we can always read at least one character after the '-'
const uint8_t *p = src + uint8_t(negative);
if(p == srcend) { return false; }
if(*p == '0') {
++p;
if(p == srcend) { return true; }
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
return true;
}
while(p != srcend && is_digit(*p)) { ++p; }
if(p == srcend) { return true; }
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
return true;
}
simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) {
// It is possible that the integer had an overflow.
// We have to handle the case where we have 0.0000somenumber.
@@ -34595,6 +34823,18 @@ simdjson_inline error_code write_float(const uint8_t *const src, bool negative,
return SUCCESS;
}
// parse the number at src
// define JSON_TEST_NUMBERS for unit testing
//
// It is assumed that the number is followed by a structural ({,},],[) character
// or a white space character. If that is not the case (e.g., when the JSON
// document is made of a single number), then it is necessary to copy the
// content and append a space before calling this function.
//
// Our objective is accurate parsing (ULP of 0) at high speed.
template<typename W>
simdjson_inline error_code parse_number(const uint8_t *const src, W &writer);
// for performance analysis, it is sometimes useful to skip parsing
#ifdef SIMDJSON_SKIPNUMBERPARSING
@@ -34673,11 +34913,11 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
// The longest positive 64-bit number is 20 digits.
// We do it this way so we don't trigger this branch unless we must.
size_t longest_digit_count = negative ? 19 : 20;
if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); }
if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); }
if (digit_count == longest_digit_count) {
if (negative) {
// Anything negative above INT64_MAX+1 is invalid
if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src); }
if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); }
WRITE_INTEGER(~i+1, src, writer);
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
return SUCCESS;
@@ -35147,19 +35387,32 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
src += uint8_t(negative);
const uint8_t *p = src;
while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
size_t digit_count = size_t(p - src);
if ( p == src ) { return NUMBER_ERROR; }
if (jsoncharutils::is_structural_or_whitespace(*p)) {
static const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
// We have an integer.
if(simdjson_unlikely(digit_count > 20)) {
return number_type::big_integer;
}
// If the number is negative and valid, it must be a signed integer.
if(negative) { return number_type::signed_integer; }
if(negative) {
if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer;
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
return number_type::big_integer;
}
return number_type::signed_integer;
}
// Let us check if we have a big integer (>=2**64).
static const uint8_t * two_to_sixtyfour = reinterpret_cast<const uint8_t *>("18446744073709551616");
if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) {
return number_type::big_integer;
}
// The number is positive and smaller than 18446744073709551616 (or 2**64).
// We want values larger or equal to 9223372036854775808 to be unsigned
// integers, and the other values to be signed integers.
int digit_count = int(p - src);
if(digit_count >= 19) {
const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
return number_type::unsigned_integer;
}
if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) {
return number_type::unsigned_integer;
}
return number_type::signed_integer;
}
@@ -35337,6 +35590,7 @@ inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
case number_type::big_integer: out << "big integer"; break;
default: SIMDJSON_UNREACHABLE();
}
return out;
@@ -38774,8 +39028,8 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
// Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD)
constexpr uint32_t substitution_code_point = 0xfffd;
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
// conversion isn't valid; we defer the check for this to inside the
// multilingual plane check
// conversion is not valid; we defer the check for this to inside the
// multilingual plane check.
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
*src_ptr += 6;
@@ -38832,8 +39086,8 @@ simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr,
// It is not ideal that this function is nearly identical to handle_unicode_codepoint.
//
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
// conversion isn't valid; we defer the check for this to inside the
// multilingual plane check
// conversion is not valid; we defer the check for this to inside the
// multilingual plane check.
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
*src_ptr += 6;
// If we found a high surrogate, we must
@@ -39768,7 +40022,8 @@ class dom_parser_implementation;
enum class number_type {
floating_point_number=1, /// a binary64 number
signed_integer, /// a signed integer that fits in a 64-bit word using two's complement
unsigned_integer /// a positive integer larger or equal to 1<<63
unsigned_integer, /// a positive integer larger or equal to 1<<63
big_integer /// a big integer that does not fit in a 64-bit word
};
} // namespace fallback
@@ -40216,11 +40471,13 @@ namespace numberparsing {
#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE)))
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE)))
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE)))
#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR)
#else
#define INVALID_NUMBER(SRC) (NUMBER_ERROR)
#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE))
#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE))
#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE))
#define BIGINT_NUMBER(SRC) (BIGINT_ERROR)
#endif
namespace {
@@ -40551,6 +40808,10 @@ simdjson_inline bool parse_digit(const uint8_t c, I &i) {
return true;
}
simdjson_inline bool is_digit(const uint8_t c) {
return static_cast<uint8_t>(c - '0') <= 9;
}
simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
// we continue with the fiction that we have an integer. If the
// floating point number is representable as x * 10^z for some integer
@@ -40630,6 +40891,23 @@ simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const s
return SUCCESS;
}
simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) {
const uint8_t *const srcend = src + max_length;
bool negative = (*src == '-'); // we can always read at least one character after the '-'
const uint8_t *p = src + uint8_t(negative);
if(p == srcend) { return false; }
if(*p == '0') {
++p;
if(p == srcend) { return true; }
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
return true;
}
while(p != srcend && is_digit(*p)) { ++p; }
if(p == srcend) { return true; }
if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; }
return true;
}
simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) {
// It is possible that the integer had an overflow.
// We have to handle the case where we have 0.0000somenumber.
@@ -40704,6 +40982,18 @@ simdjson_inline error_code write_float(const uint8_t *const src, bool negative,
return SUCCESS;
}
// parse the number at src
// define JSON_TEST_NUMBERS for unit testing
//
// It is assumed that the number is followed by a structural ({,},],[) character
// or a white space character. If that is not the case (e.g., when the JSON
// document is made of a single number), then it is necessary to copy the
// content and append a space before calling this function.
//
// Our objective is accurate parsing (ULP of 0) at high speed.
template<typename W>
simdjson_inline error_code parse_number(const uint8_t *const src, W &writer);
// for performance analysis, it is sometimes useful to skip parsing
#ifdef SIMDJSON_SKIPNUMBERPARSING
@@ -40782,11 +41072,11 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
// The longest positive 64-bit number is 20 digits.
// We do it this way so we don't trigger this branch unless we must.
size_t longest_digit_count = negative ? 19 : 20;
if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); }
if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); }
if (digit_count == longest_digit_count) {
if (negative) {
// Anything negative above INT64_MAX+1 is invalid
if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src); }
if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); }
WRITE_INTEGER(~i+1, src, writer);
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
return SUCCESS;
@@ -41256,19 +41546,32 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
src += uint8_t(negative);
const uint8_t *p = src;
while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
size_t digit_count = size_t(p - src);
if ( p == src ) { return NUMBER_ERROR; }
if (jsoncharutils::is_structural_or_whitespace(*p)) {
static const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
// We have an integer.
if(simdjson_unlikely(digit_count > 20)) {
return number_type::big_integer;
}
// If the number is negative and valid, it must be a signed integer.
if(negative) { return number_type::signed_integer; }
if(negative) {
if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer;
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
return number_type::big_integer;
}
return number_type::signed_integer;
}
// Let us check if we have a big integer (>=2**64).
static const uint8_t * two_to_sixtyfour = reinterpret_cast<const uint8_t *>("18446744073709551616");
if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) {
return number_type::big_integer;
}
// The number is positive and smaller than 18446744073709551616 (or 2**64).
// We want values larger or equal to 9223372036854775808 to be unsigned
// integers, and the other values to be signed integers.
int digit_count = int(p - src);
if(digit_count >= 19) {
const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
if((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) {
return number_type::unsigned_integer;
}
if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) {
return number_type::unsigned_integer;
}
return number_type::signed_integer;
}
@@ -41446,6 +41749,7 @@ inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept {
case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break;
case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break;
case number_type::floating_point_number: out << "floating-point number (binary64)"; break;
case number_type::big_integer: out << "big integer"; break;
default: SIMDJSON_UNREACHABLE();
}
return out;
@@ -41966,8 +42270,8 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
// Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD)
constexpr uint32_t substitution_code_point = 0xfffd;
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
// conversion isn't valid; we defer the check for this to inside the
// multilingual plane check
// conversion is not valid; we defer the check for this to inside the
// multilingual plane check.
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
*src_ptr += 6;
@@ -42024,8 +42328,8 @@ simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr,
// It is not ideal that this function is nearly identical to handle_unicode_codepoint.
//
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
// conversion isn't valid; we defer the check for this to inside the
// multilingual plane check
// conversion is not valid; we defer the check for this to inside the
// multilingual plane check.
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
*src_ptr += 6;
// If we found a high surrogate, we must
+1008 -285
View File
File diff suppressed because it is too large Load Diff
+4 -4
View File
@@ -53,8 +53,8 @@ simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
// Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD)
constexpr uint32_t substitution_code_point = 0xfffd;
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
// conversion isn't valid; we defer the check for this to inside the
// multilingual plane check
// conversion is not valid; we defer the check for this to inside the
// multilingual plane check.
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
*src_ptr += 6;
@@ -111,8 +111,8 @@ simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr,
// It is not ideal that this function is nearly identical to handle_unicode_codepoint.
//
// jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
// conversion isn't valid; we defer the check for this to inside the
// multilingual plane check
// conversion is not valid; we defer the check for this to inside the
// multilingual plane check.
uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
*src_ptr += 6;
// If we found a high surrogate, we must
+1
View File
@@ -18,6 +18,7 @@ namespace internal {
{ F_ATOM_ERROR, "F_ATOM_ERROR: Problem while parsing an atom starting with the letter 'f'" },
{ N_ATOM_ERROR, "N_ATOM_ERROR: Problem while parsing an atom starting with the letter 'n'" },
{ NUMBER_ERROR, "NUMBER_ERROR: Problem while parsing a number" },
{ BIGINT_ERROR, "BIGINT_ERROR: Big integer value that cannot be represented using 64 bits" },
{ UTF8_ERROR, "UTF8_ERROR: The input is not valid UTF-8" },
{ UNINITIALIZED, "UNINITIALIZED: Uninitialized" },
{ EMPTY, "EMPTY: no JSON found" },
@@ -102,7 +102,7 @@ namespace array_error_tests {
ONDEMAND_SUBTEST("unclosed extra comma", R"({ "a": [,)", assert_iterate(doc["a"], { INCOMPLETE_ARRAY_OR_OBJECT }));
ONDEMAND_SUBTEST("unclosed extra comma", R"({ "a": [,,)", assert_iterate(doc["a"], { INCOMPLETE_ARRAY_OR_OBJECT }));
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1 )", assert_iterate(doc["a"], { INCOMPLETE_ARRAY_OR_OBJECT }));
// TODO These pass the user values that may run past the end of the buffer if they aren't careful
// TODO These pass the user values that may run past the end of the buffer if they are not careful
// In particular, if the padding is decorated with the wrong values, we could cause overrun!
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1,)", assert_iterate(doc["a"], { INCOMPLETE_ARRAY_OR_OBJECT }));
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1)", assert_iterate(doc["a"], { INCOMPLETE_ARRAY_OR_OBJECT }));
@@ -651,6 +651,26 @@ namespace document_stream_tests {
}
bool issue2137() {
TEST_START();
auto json = "true { "_padded;
ondemand::parser parser;
ondemand::document_stream stream;
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
for (auto doc: stream) {
bool val{};
ASSERT_SUCCESS(doc.get_bool().get(val));
std::string_view raw_json;
ASSERT_SUCCESS(doc.raw_json_token().get(raw_json));
ASSERT_EQUAL(val, 1);
std::string s(raw_json);
ASSERT_EQUAL(s, "true ");
}
size_t t = stream.truncated_bytes();
ASSERT_EQUAL(3, t);
TEST_SUCCEED();
}
bool issue1668() {
TEST_START();
auto json = R"([1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100])"_padded;
@@ -861,6 +881,7 @@ namespace document_stream_tests {
bool run() {
return
issue2137() &&
skipbom() &&
issue1977() &&
string_with_trailing() &&
+25
View File
@@ -106,6 +106,29 @@ namespace error_tests {
TEST_SUCCEED();
}
#if SIMDJSON_EXCEPTIONS
// This is a compile-only test
struct Class {
Class(std::string text)
: parser()
, doc(parser.iterate(text))
{
}
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc;
};
bool document_in_class() {
TEST_START();
std::string text = "{}";
Class c(text);
TEST_SUCCEED();
}
bool direct_document() {
TEST_START();
std::string text = "{}";
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc(parser.iterate(text));
TEST_SUCCEED();
}
bool raw_json_string_except() {
TEST_START();
ondemand::parser parser;
@@ -381,6 +404,8 @@ namespace error_tests {
issue1834() &&
issue1834_2() &&
#if SIMDJSON_EXCEPTIONS
document_in_class() &&
direct_document() &&
raw_json_string_except() &&
raw_json_string_except_with_io() &&
#endif
+96 -31
View File
@@ -211,7 +211,7 @@ namespace number_tests {
bool get_number_tests() {
TEST_START();
ondemand::parser parser;
padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999,-9223372036854775807,-9223372036854775808])"_padded;
padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999,-9223372036854775807,-9223372036854775808,12345678901234567890123])"_padded;
ondemand::number_type expectedtypes[] = {ondemand::number_type::floating_point_number,
ondemand::number_type::signed_integer,
ondemand::number_type::signed_integer,
@@ -219,10 +219,11 @@ namespace number_tests {
ondemand::number_type::signed_integer,
ondemand::number_type::unsigned_integer,
ondemand::number_type::signed_integer,
ondemand::number_type::signed_integer
ondemand::number_type::signed_integer,
ondemand::number_type::big_integer
};
bool is_negative[] = {false, false, false, false, true, false, true, true};
bool is_integer[] = {false, true, true, false, true, true, true, true};
bool is_negative[] = {false, false, false, false, true, false, true, true, false};
bool is_integer[] = {false, true, true, false, true, true, true, true, true};
ondemand::document doc;
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
@@ -233,17 +234,25 @@ namespace number_tests {
ondemand::value val;
ASSERT_SUCCESS(valr.get(val));
ondemand::number_type nt{};
ASSERT_SUCCESS(val.get_number_type().get(nt));
auto r = val.get_number_type().get(nt);
ASSERT_SUCCESS(r);
ASSERT_EQUAL(expectedtypes[counter], nt);
ondemand::number num;
ASSERT_SUCCESS(val.get_number().get(num));
ASSERT_EQUAL(is_negative[counter], val.is_negative());
bool intvalue{};
ASSERT_SUCCESS(val.is_integer().get(intvalue));
auto res = val.is_integer().get(intvalue);
ASSERT_SUCCESS(res);
ondemand::number num;
auto rr = val.get_number().get(num);
if(counter == 8) {
ASSERT_EQUAL(rr, BIGINT_ERROR);
ASSERT_EQUAL(nt, ondemand::number_type::big_integer);
} else {
ASSERT_SUCCESS(rr);
ondemand::number_type t = num.get_number_type();
ASSERT_EQUAL(expectedtypes[counter], t);
}
ASSERT_EQUAL(is_integer[counter], intvalue);
ondemand::number_type t = num.get_number_type();
ASSERT_EQUAL(expectedtypes[counter], t);
switch(t) {
ASSERT_EQUAL(is_negative[counter], val.is_negative());
switch(nt) {
case ondemand::number_type::signed_integer:
ASSERT_TRUE(num.is_int64());
break;
@@ -253,25 +262,39 @@ namespace number_tests {
case ondemand::number_type::floating_point_number:
ASSERT_TRUE(num.is_double());
break;
case ondemand::number_type::big_integer:
break;
}
if(counter == 0) {
ASSERT_EQUAL(num.get_double(), 1.0);
ASSERT_EQUAL((double)num, 1.0);
} else if(counter == 1) {
ASSERT_EQUAL(num.get_int64(), 3);
ASSERT_EQUAL((int64_t)num, 3);
} else if(counter == 2) {
ASSERT_EQUAL(num.get_int64(), 1);
ASSERT_EQUAL((int64_t)num, 1);
} else if(counter == 3) {
ASSERT_EQUAL(num.get_double(), 3.1415);
ASSERT_EQUAL((double)num, 3.1415);
} else if(counter == 4) {
ASSERT_EQUAL(num.get_int64(), -13231232);
ASSERT_EQUAL((int64_t)num, -13231232);
} else if(counter == 5) {
ASSERT_EQUAL(num.get_uint64(), UINT64_C(9999999999999999999));
ASSERT_EQUAL((uint64_t)num, UINT64_C(9999999999999999999));
switch(counter) {
case 0:
ASSERT_EQUAL(num.get_double(), 1.0);
ASSERT_EQUAL((double)num, 1.0);
break;
case 1:
ASSERT_EQUAL(num.get_int64(), 3);
ASSERT_EQUAL((int64_t)num, 3);
break;
case 2:
ASSERT_EQUAL(num.get_int64(), 1);
ASSERT_EQUAL((int64_t)num, 1);
break;
case 3:
ASSERT_EQUAL(num.get_double(), 3.1415);
ASSERT_EQUAL((double)num, 3.1415);
break;
case 4:
ASSERT_EQUAL(num.get_int64(), -13231232);
ASSERT_EQUAL((int64_t)num, -13231232);
break;
case 5:
ASSERT_EQUAL(num.get_uint64(), UINT64_C(9999999999999999999));
ASSERT_EQUAL((uint64_t)num, UINT64_C(9999999999999999999));
break;
case 8:
ASSERT_EQUAL(val.raw_json_token(), "12345678901234567890123");
break;
default:
break;
}
counter++;
}
@@ -441,8 +464,50 @@ namespace number_tests {
TEST_SUCCEED();
}
bool big_int_not_zero() {
TEST_START();
ondemand::parser parser;
ondemand::document doc;
// This is not a big integer, it is a mistake
padded_string docdata = R"(09500000000000000000000000000000000000)"_padded;
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
ASSERT_ERROR(doc.get_number(), NUMBER_ERROR);
TEST_SUCCEED();
}
bool negative_big_int() {
TEST_START();
ondemand::parser parser;
ondemand::document doc;
// This is not a big integer, it is a mistake
padded_string docdata = R"(-18446744073709551616)"_padded;
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
ASSERT_ERROR(doc.get_number(), BIGINT_ERROR);
std::string_view my_big;
ASSERT_SUCCESS(doc.raw_json_token().get(my_big));
ASSERT_EQUAL(my_big, "-18446744073709551616");
TEST_SUCCEED();
}
bool gigantic_big_int() {
TEST_START();
ondemand::parser parser;
ondemand::document doc;
std::string number(2000, '1');
// This is not a big integer, it is a mistake
ASSERT_SUCCESS(parser.iterate(number).get(doc));
ASSERT_ERROR(doc.get_number(), BIGINT_ERROR);
std::string_view my_big;
ASSERT_SUCCESS(doc.raw_json_token().get(my_big));
ASSERT_EQUAL(my_big, number);
TEST_SUCCEED();
}
bool run() {
return issue2099() &&
return gigantic_big_int() &&
big_int_not_zero() &&
negative_big_int() &&
issue2099() &&
issue2093() &&
issue2045() &&
issue2017() &&
@@ -119,7 +119,7 @@ namespace object_error_tests {
bool object_lookup_miss_unclosed_error() {
TEST_START();
ONDEMAND_SUBTEST("unclosed", R"({ "a": 1, )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT));
// TODO These next two pass the user a value that may run past the end of the buffer if they aren't careful.
// TODO These next two pass the user a value that may run past the end of the buffer if they are not careful.
// In particular, if the padding is decorated with the wrong values, we could cause overrun!
ONDEMAND_SUBTEST("unclosed", R"({ "a": 1 )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT));
ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT));
+78 -6
View File
@@ -3,6 +3,9 @@
#if __cpp_lib_optional >= 201606L
#include <optional>
#endif
#if SIMDJSON_CPLUSPLUS17
#include <charconv>
#endif
using namespace std;
using namespace simdjson;
using error_code=simdjson::error_code;
@@ -382,22 +385,84 @@ bool examplecrt_realloc() {
TEST_SUCCEED();
}
#if SIMDJSON_CPLUSPLUS17
bool big_int_array() {
TEST_START();
ondemand::parser parser;
padded_string docdata = R"([-9223372036854775809, 18446744073709551617, 99999999999999999999999 ])"_padded;
std::string expected[] = {"-9223372036854775809", "18446744073709551617", "99999999999999999999999 "};
ondemand::document doc = parser.iterate(docdata);
ondemand::array arr = doc.get_array();
size_t i = 0;
for(ondemand::value val : arr) {
if(i > 3) {
std::cerr << "unexpected number of elements" << std::endl;
return false;
}
if(val.get_number_type() != ondemand::number_type::big_integer) {
std::cerr << "unexpected number type" << std::endl;
std::cout << val.get_number_type() << std::endl;
std::cout << val.raw_json_token() << std::endl;
return false;
}
std::string_view token = val.raw_json_token();
std::string_view expected_token = expected[i];
if(token != expected_token) {
std::cerr << "unexpected token: " << token << " expected: " << expected_token << std::endl;
return false;
}
i++;
}
if(i != 3) {
std::cerr << "unexpected number of elements" << std::endl;
return false;
}
TEST_SUCCEED();
}
#endif
bool big_int_array_as_double() {
TEST_START();
ondemand::parser parser;
padded_string docdata = R"([-9223372036854775809, 18446744073709551617, 99999999999999999999999 ])"_padded;
double dexpected[] = {-9223372036854775808.0, 18446744073709551616.0, 1e23};
ondemand::document doc = parser.iterate(docdata);
ondemand::array arr = doc.get_array();
size_t i = 0;
for(ondemand::value val : arr) {
if(i > 3) {
std::cerr << "unexpected number of elements" << std::endl;
return false;
}
if((val.get_number_type() != ondemand::number_type::big_integer) || (dexpected[i] != val.get_double())) {
return false;
}
i++;
}
if(i != 3) {
std::cerr << "unexpected number of elements" << std::endl;
return false;
}
TEST_SUCCEED();
}
bool number_tests() {
TEST_START();
ondemand::parser parser;
padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999])"_padded;
padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999,12345678901234567890123])"_padded;
ondemand::document doc = parser.iterate(docdata);
ondemand::array arr = doc.get_array();
for(ondemand::value val : arr) {
std::cout << val << " ";
std::cout << "negative: " << val.is_negative() << " ";
std::cout << "is_integer: " << val.is_integer() << " ";
ondemand::number num = val.get_number();
ondemand::number_type t = num.get_number_type();
// direct computation without materializing the number:
ondemand::number_type dt = val.get_number_type();
if(t != dt) { throw std::runtime_error("bug"); }
switch(t) {
ondemand::number num;
auto res = val.get_number().get(num);
ondemand::number_type t = num.get_number_type();
if(t != dt && res != BIGINT_ERROR) { throw std::runtime_error("bug"); }
switch(dt) {
case ondemand::number_type::signed_integer:
std::cout << "integer: " << int64_t(num) << " ";
std::cout << "integer: " << num.get_int64() << std::endl;
@@ -410,6 +475,9 @@ bool number_tests() {
std::cout << "float: " << double(num) << " ";
std::cout << "float: " << num.get_double() << std::endl;
break;
case ondemand::number_type::big_integer:
std::cout << "big-integer: " << val.raw_json_token() << std::endl;
break;
}
}
TEST_SUCCEED();
@@ -1778,6 +1846,10 @@ bool value_raw_json_object() {
bool run() {
return true
#if SIMDJSON_EXCEPTIONS
#if SIMDJSON_CPLUSPLUS17
&& big_int_array()
#endif // SIMDJSON_CPLUSPLUS17
&& big_int_array_as_double()
&& key_raw_json_token()
&& to_optional()
&& value_raw_json_array() && value_raw_json_object()
@@ -1785,7 +1857,7 @@ bool run() {
&& at_end()
&& example1956() && example1958()
&& allow_comma_separated_example()
// && basics_1() // Fails because twitter.json isn't in current directory. Compile test only.
// && basics_1() // Fails because twitter.json is not in current directory. Compile test only.
&& basics_treewalk()
&& basics_treewalk_breakline()
&& json_value_with_array_count()
+7 -7
View File
@@ -16,7 +16,7 @@ def colored(r, g, b, text):
return "\033[38;2;{};{};{}m{} \033[38;2;255;255;255m".format(r, g, b, text)
def extractnumbers(s):
return tuple(map(int,re.findall("(\d+)\.(\d+)\.(\d+)",str(s))[0]))
return tuple(map(int,re.findall(r"(\d+)\.(\d+)\.(\d+)",str(s))[0]))
def toversionstring(major, minor, rev):
return str(major)+"."+str(minor)+"."+str(rev)
@@ -123,7 +123,7 @@ newversionstring = str(newversion[0]) + "." + str(newversion[1]) + "." + str(new
cmakefile = maindir + os.sep + "CMakeLists.txt"
sonumber = None
pattern = re.compile("set\(SIMDJSON_LIB_SOVERSION \"(\d+)\" CACHE STRING \"simdjson library soversion\"\)")
pattern = re.compile(r"set\(SIMDJSON_LIB_SOVERSION \"(\d+)\" CACHE STRING \"simdjson library soversion\"\)")
with open (cmakefile, 'rt') as myfile:
for line in myfile:
m = pattern.search(line)
@@ -138,9 +138,9 @@ if(atleastminor):
sonumber += 1
for line in fileinput.input(cmakefile, inplace=1, backup='.bak'):
line = re.sub(' VERSION \d+\.\d+\.\d+',' VERSION '+newmajorversionstring+'.'+mewminorversionstring+'.'+newrevversionstring, line.rstrip())
line = re.sub('SIMDJSON_LIB_VERSION "\d+','SIMDJSON_LIB_VERSION "'+str(sonumber), line)
line = re.sub('set\(SIMDJSON_LIB_SOVERSION \"\d+\"','set(SIMDJSON_LIB_SOVERSION \"'+str(sonumber)+'\"', line)
line = re.sub(r' VERSION \d+\.\d+\.\d+',' VERSION '+newmajorversionstring+'.'+mewminorversionstring+'.'+newrevversionstring, line.rstrip())
line = re.sub(r'SIMDJSON_LIB_VERSION "\d+','SIMDJSON_LIB_VERSION "'+str(sonumber), line)
line = re.sub(r'set\(SIMDJSON_LIB_SOVERSION \"\d+\"','set(SIMDJSON_LIB_SOVERSION \"'+str(sonumber)+'\"', line)
print(line)
print("modified "+cmakefile+", a backup was made")
@@ -148,7 +148,7 @@ print("modified "+cmakefile+", a backup was made")
doxyfile = maindir + os.sep + "Doxyfile"
for line in fileinput.input(doxyfile, inplace=1, backup='.bak'):
line = re.sub('PROJECT_NUMBER = "\d+\.\d+\.\d+','PROJECT_NUMBER = "'+newversionstring, line.rstrip())
line = re.sub(r'PROJECT_NUMBER = "\d+\.\d+\.\d+','PROJECT_NUMBER = "'+newversionstring, line.rstrip())
print(line)
print("modified "+doxyfile+", a backup was made")
@@ -168,7 +168,7 @@ if(cp.returncode != 0):
print("Failed to run doxygen")
pattern = re.compile("https://simdjson.org/api/(\d+\.\d+\.\d+)/index.html")
pattern = re.compile(r"https://simdjson.org/api/(\d+\.\d+\.\d+)/index.html")
readmefile = maindir + os.sep + "README.md"
readmedata = open(readmefile).read()
m = pattern.search(readmedata)