From b9b20be80e9e89c77fb5e7b27d69896000da31cd Mon Sep 17 00:00:00 2001 From: Alecto Irene Perez Date: Mon, 4 May 2026 15:24:26 -0400 Subject: [PATCH] Add support for parsing NaN and Infinity as requested in #1540, #2414, and #2540 (#2696) * add compile option 'SIMDJSON_ENABLE_NAN_INF' but disable by default * extend parser to support NaN/Infinity when SIMDJSON_ENABLE_NAN_INF=1 * update tests to check parsing of NaN/Infinity, when enabled * update minefield tests: mark nan/inf tests as passing when nan/inf is ON * update CI/CD to run tests with extensions for NaN/Infinity enabled --- .github/workflows/macos.yml | 11 + .github/workflows/ubuntu24-sani.yml | 14 + .github/workflows/ubuntu24.yml | 3 +- .github/workflows/vs17-ci.yml | 20 +- CMakeLists.txt | 6 + include/simdjson/common_defs.h | 4 + include/simdjson/generic/atomparsing.h | 90 ++++- include/simdjson/generic/numberparsing.h | 64 +++- src/generic/stage2/json_iterator.h | 28 +- src/generic/stage2/tape_builder.h | 43 ++- tests/dom/CMakeLists.txt | 1 + tests/dom/minefieldcheck.cpp | 26 +- tests/dom/nan_inf_tests.cpp | 311 ++++++++++++++++ tests/ondemand/CMakeLists.txt | 1 + tests/ondemand/ondemand_nan_inf_tests.cpp | 352 ++++++++++++++++++ .../ondemand_number_in_string_tests.cpp | 8 +- tests/ondemand/ondemand_unknown_tests.cpp | 21 +- 17 files changed, 978 insertions(+), 25 deletions(-) create mode 100644 tests/dom/nan_inf_tests.cpp create mode 100644 tests/ondemand/ondemand_nan_inf_tests.cpp diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 1eda9ebe7..2c6261487 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -42,3 +42,14 @@ jobs: echo -e '#include \nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json && cd ../tests/installation_tests/find && mkdir buildshared && cd buildshared && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../buildshared/destination .. && cmake --build . + - name: Use cmake (parsing for NaN/Infinity enabled) + run: | + mkdir build_nan_inf && + cd build_nan_inf && + cmake -DSIMDJSON_ENABLE_NAN_INF=ON -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && + cmake --build . && + ctest --output-on-failure -LE explicitonly -j && + cmake --install . && + echo -e '#include \nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json && + cd ../tests/installation_tests/find && + mkdir build_nan_inf && cd build_nan_inf && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build_nan_inf/destination .. && cmake --build . diff --git a/.github/workflows/ubuntu24-sani.yml b/.github/workflows/ubuntu24-sani.yml index 06875de3e..e5de40772 100644 --- a/.github/workflows/ubuntu24-sani.yml +++ b/.github/workflows/ubuntu24-sani.yml @@ -21,6 +21,13 @@ jobs: cmake -DSIMDJSON_SANITIZE=ON -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. && cmake --build . && ctest --output-on-failure -LE explicitonly -j + - name: Use cmake with address sanitizer (Parsing of NaN/Infinity enabled) + run: | + mkdir builddebug_nan_inf && + cd builddebug_nan_inf && + cmake -DSIMDJSON_SANITIZE=ON -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_NAN_INF=ON .. && + cmake --build . && + ctest --output-on-failure -LE explicitonly -j ubuntu-build-undefined-sanitizer: if: >- ! contains(toJSON(github.event.commits.*.message), '[skip ci]') && @@ -39,3 +46,10 @@ jobs: cmake -DSIMDJSON_SANITIZE_UNDEFINED=ON -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. && cmake --build . && ctest --output-on-failure -LE explicitonly -j + - name: Use cmake with undefined sanitizer (Parsing of NaN/Infinity enabled) + run: | + mkdir builddebugundefsani_nan_inf && + cd builddebugundefsani_nan_inf && + cmake -DSIMDJSON_SANITIZE_UNDEFINED=ON -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_NAN_INF=ON .. && + cmake --build . && + ctest --output-on-failure -LE explicitonly -j diff --git a/.github/workflows/ubuntu24.yml b/.github/workflows/ubuntu24.yml index 3ca0c96c9..3193c49ba 100644 --- a/.github/workflows/ubuntu24.yml +++ b/.github/workflows/ubuntu24.yml @@ -12,11 +12,12 @@ jobs: shared: [ON, OFF] cxx: [g++-13, clang++-16] sanitizer: [ON, OFF] + nan_inf: [ON, OFF] build_type: [RelWithDebInfo, Debug, Release] steps: - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - name: Prepare - run: cmake -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_SANITIZE=${{matrix.sanitizer}} -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build + run: cmake -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_SANITIZE=${{matrix.sanitizer}} -DBUILD_SHARED_LIBS=${{matrix.shared}} -DSIMDJSON_ENABLE_NAN_INF=${{matrix.nan_inf}} -B build env: CXX: ${{matrix.cxx}} - name: Build diff --git a/.github/workflows/vs17-ci.yml b/.github/workflows/vs17-ci.yml index cca21ba4f..239428a4b 100644 --- a/.github/workflows/vs17-ci.yml +++ b/.github/workflows/vs17-ci.yml @@ -13,20 +13,22 @@ jobs: fail-fast: false matrix: include: - - {gen: Visual Studio 17 2022, arch: Win32, shared: ON, build_type: Release, memory_map: OFF} - - {gen: Visual Studio 17 2022, arch: Win32, shared: OFF, build_type: Release, memory_map: OFF} - - {gen: Visual Studio 17 2022, arch: x64, shared: ON, build_type: Release, memory_map: OFF} - - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Debug, memory_map: OFF} - - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Release, memory_map: OFF} - - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: RelWithDebInfo, memory_map: OFF} + - {gen: Visual Studio 17 2022, arch: Win32, shared: ON, build_type: Release, memory_map: OFF, nan_inf: OFF} + - {gen: Visual Studio 17 2022, arch: Win32, shared: OFF, build_type: Release, memory_map: OFF, nan_inf: OFF} + - {gen: Visual Studio 17 2022, arch: x64, shared: ON, build_type: Release, memory_map: OFF, nan_inf: OFF} + - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Debug, memory_map: OFF, nan_inf: OFF} + - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Release, memory_map: OFF, nan_inf: OFF} + - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: RelWithDebInfo, memory_map: OFF, nan_inf: OFF} + - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Debug, memory_map: OFF, nan_inf: ON} + - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Release, memory_map: OFF, nan_inf: ON} # Exercise the opt-in Windows memory-file mapping path at least once in CI. - - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Release, memory_map: ON} + - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Release, memory_map: ON, nan_inf: OFF} steps: - name: checkout uses: actions/checkout@v4 - name: Configure run: | - cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -DSIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=${{matrix.memory_map}} -B build + cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -DSIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=${{matrix.memory_map}} -DSIMDJSON_ENABLE_NAN_INF=${{matrix.nan_inf}} -B build - name: Build Debug run: cmake --build build --config ${{matrix.build_type}} --verbose - name: Run tests @@ -39,4 +41,4 @@ jobs: - name: Test Installation run: | cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -B build_install_test tests/installation_tests/find - cmake --build build_install_test --config ${{matrix.build_type}} \ No newline at end of file + cmake --build build_install_test --config ${{matrix.build_type}} diff --git a/CMakeLists.txt b/CMakeLists.txt index 743bc7e8e..5c1227ae3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -254,6 +254,12 @@ if(SIMDJSON_MINUS_ZERO_AS_FLOAT) simdjson_add_props(target_compile_definitions PRIVATE SIMDJSON_MINUS_ZERO_AS_FLOAT=1) endif(SIMDJSON_MINUS_ZERO_AS_FLOAT) +option(SIMDJSON_ENABLE_NAN_INF "Allow parsing of NaN and Infinity JSON values" OFF) +if(SIMDJSON_ENABLE_NAN_INF) + message(STATUS "simdjson NaN and Infinity parsing is enabled.") + simdjson_add_props(target_compile_definitions PUBLIC SIMDJSON_ENABLE_NAN_INF=1) +endif() + # GCC and Clang have horrendous Debug builds when using SIMD. # A common fix is to use '-Og' instead. # bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412 diff --git a/include/simdjson/common_defs.h b/include/simdjson/common_defs.h index 1a2c7c9f3..aa841e423 100644 --- a/include/simdjson/common_defs.h +++ b/include/simdjson/common_defs.h @@ -30,6 +30,10 @@ double from_chars(const char *first, const char* end) noexcept; #endif #endif +#ifndef SIMDJSON_ENABLE_NAN_INF +#define SIMDJSON_ENABLE_NAN_INF 0 +#endif + } // namespace simdjson #if defined(__GNUC__) diff --git a/include/simdjson/generic/atomparsing.h b/include/simdjson/generic/atomparsing.h index 2eeb7b81a..598c5b079 100644 --- a/include/simdjson/generic/atomparsing.h +++ b/include/simdjson/generic/atomparsing.h @@ -22,6 +22,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -33,6 +36,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -69,9 +94,72 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8) { return str8ncmp_case_insensitive(src, "infinity") == 0; } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -#endif // SIMDJSON_GENERIC_ATOMPARSING_H \ No newline at end of file +#endif // SIMDJSON_GENERIC_ATOMPARSING_H diff --git a/include/simdjson/generic/numberparsing.h b/include/simdjson/generic/numberparsing.h index 5ebcb52f2..35b02dc17 100644 --- a/include/simdjson/generic/numberparsing.h +++ b/include/simdjson/generic/numberparsing.h @@ -4,6 +4,7 @@ #define SIMDJSON_GENERIC_NUMBERPARSING_H #include "simdjson/generic/base.h" #include "simdjson/generic/jsoncharutils.h" +#include "simdjson/generic/atomparsing.h" #include "simdjson/internal/numberparsing_tables.h" #endif // SIMDJSON_CONDITIONAL_INCLUDE @@ -299,6 +300,24 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -600,7 +619,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -1031,7 +1064,17 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 bool leading_zero = (i == 0); while (parse_digit(*p, i)) { p++; } // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -1249,7 +1292,22 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c bool leading_zero = (i == 0); while (parse_digit(*p, i)) { p++; } // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // diff --git a/src/generic/stage2/json_iterator.h b/src/generic/stage2/json_iterator.h index 810e8fc52..32f087cb6 100644 --- a/src/generic/stage2/json_iterator.h +++ b/src/generic/stage2/json_iterator.h @@ -291,7 +291,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -313,7 +326,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -325,4 +351,4 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H \ No newline at end of file +#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H diff --git a/src/generic/stage2/tape_builder.h b/src/generic/stage2/tape_builder.h index 58041aa99..5cd1e90a8 100644 --- a/src/generic/stage2/tape_builder.h +++ b/src/generic/stage2/tape_builder.h @@ -76,6 +76,15 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; @@ -255,6 +264,38 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_at return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value)) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value)) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { @@ -310,4 +351,4 @@ simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H \ No newline at end of file +#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H diff --git a/tests/dom/CMakeLists.txt b/tests/dom/CMakeLists.txt index 09b8b9cb0..cc66a0303 100644 --- a/tests/dom/CMakeLists.txt +++ b/tests/dom/CMakeLists.txt @@ -13,6 +13,7 @@ add_cpp_test(errortests LABELS dom acceptance per_implementation add_cpp_test(extracting_values_example LABELS dom acceptance per_implementation) add_cpp_test(integer_tests LABELS dom acceptance per_implementation) add_cpp_test(big_integer_tests LABELS dom acceptance per_implementation) +add_cpp_test(nan_inf_tests LABELS dom acceptance per_implementation) add_cpp_test(jsoncheck LABELS dom acceptance per_implementation) add_cpp_test(json_path_tests LABELS dom acceptance per_implementation) add_cpp_test(minefieldcheck LABELS dom acceptance per_implementation) diff --git a/tests/dom/minefieldcheck.cpp b/tests/dom/minefieldcheck.cpp index e2d1e9db2..6adeb0368 100644 --- a/tests/dom/minefieldcheck.cpp +++ b/tests/dom/minefieldcheck.cpp @@ -60,6 +60,26 @@ bool validate_minefield(const char *dirname) { char *fullpath = static_cast(malloc(fullpathlen)); snprintf(fullpath, fullpathlen, "%s%s%s", dirname, needsep ? "/" : "", name); + // Skip any files that have invalid names + if (namelen < 2 || name[1] != '_') { + printf("warning: file %s should begin with 'y_', 'n_', or 'i_' (skipping)\n", name); + continue; + } + + // Determines if the file should pass. + // 'y' for 'expected to pass', 'n' for 'expected to fail', 'i' for 'ignore' + char should_pass = name[0]; + +#if SIMDJSON_ENABLE_NAN_INF + // If nan/infinity are enabled, these files should pass (rather than failing). + // + // Therefore, we mark them as 'should_pass' + bool is_nan_inf_test = contains("NaN", name) || contains("_Inf", name) || contains("_infinity", name); + if (is_nan_inf_test && should_pass == 'n') { + should_pass = 'y'; + } +#endif + simdjson::padded_string p; auto error = simdjson::padded_string::load(fullpath).get(p); if (error) { @@ -72,16 +92,16 @@ bool validate_minefield(const char *dirname) { auto errorcode = parser.parse(p).error(); ++how_many; printf("%s\n", errorcode == simdjson::error_code::SUCCESS ? "ok" : "invalid"); - if (starts_with("i_", name) ) { + if (should_pass == 'i') { // skipping how_many--; - } else if (starts_with("y_", name) && errorcode != simdjson::error_code::SUCCESS) { + } else if (should_pass == 'y' && errorcode != simdjson::error_code::SUCCESS) { is_file_as_expected[i] = false; printf("warning: file %s should pass but it fails. Error is: %s\n", name, simdjson::error_message(errorcode)); printf("size of file in bytes: %zu \n", p.size()); everything_fine = false; - } else if (starts_with("n_", name) && errorcode == simdjson::error_code::SUCCESS) { + } else if (should_pass == 'n' && errorcode == simdjson::error_code::SUCCESS) { is_file_as_expected[i] = false; printf("warning: file %s should fail but it passes.\n", name); printf("size of file in bytes: %zu \n", p.size()); diff --git a/tests/dom/nan_inf_tests.cpp b/tests/dom/nan_inf_tests.cpp new file mode 100644 index 000000000..038e31a7f --- /dev/null +++ b/tests/dom/nan_inf_tests.cpp @@ -0,0 +1,311 @@ +#include "simdjson.h" +#include "test_macros.h" +#include "test_main.h" +#include +#include + +using namespace simdjson; + +namespace nan_inf_tests { +#if SIMDJSON_ENABLE_NAN_INF + +bool parse_nan() { + TEST_START(); + for (auto json_str : {"NaN", "nan", "NAN"}) { + dom::parser parser; + dom::element doc; + ASSERT_SUCCESS( + parser.parse(padded_string(json_str, strlen(json_str))).get(doc)); + double value; + ASSERT_SUCCESS(doc.get_double().get(value)); + ASSERT_TRUE(std::isnan(value)); + } + TEST_SUCCEED(); +} + +bool parse_infinity() { + TEST_START(); + for (auto json_str : + {"infinity", "Infinity", "INFINITY", "inf", "Inf", "INF"}) { + dom::parser parser; + dom::element doc; + ASSERT_SUCCESS( + parser.parse(padded_string(json_str, strlen(json_str))).get(doc)); + double value; + ASSERT_SUCCESS(doc.get_double().get(value)); + ASSERT_TRUE(std::isinf(value)); + ASSERT_TRUE(value > 0); + } + TEST_SUCCEED(); +} + +bool parse_negative_infinity() { + TEST_START(); + for (auto json_str : + {"-infinity", "-Infinity", "-INFINITY", "-inf", "-Inf", "-INF"}) { + dom::parser parser; + dom::element doc; + ASSERT_SUCCESS( + parser.parse(padded_string(json_str, strlen(json_str))).get(doc)); + double value; + ASSERT_SUCCESS(doc.get_double().get(value)); + ASSERT_TRUE(std::isinf(value)); + ASSERT_TRUE(value < 0); + } + TEST_SUCCEED(); +} + +bool nan_in_array() { + TEST_START(); + dom::parser parser; + dom::element doc; + auto json = R"([5, NaN, NAN, nan, -nan, -NAN, -NaN, 1.25])"_padded; + ASSERT_SUCCESS(parser.parse(json).get(doc)); + dom::array arr; + ASSERT_SUCCESS(doc.get_array().get(arr)); + + double nan = std::numeric_limits::quiet_NaN(); + std::array expected_values{5, nan, nan, nan, nan, nan, nan, 1.25}; + + size_t index = 0; + for (auto val : arr) { + if (index == expected_values.size()) { + TEST_FAIL("Array contained more values than expected"); + } + + double parsed; + ASSERT_SUCCESS(val.get_double().get(parsed)); + double expected = expected_values[index]; + if (std::isnan(expected)) { + ASSERT_TRUE(std::isnan(parsed)) + } else { + ASSERT_EQUAL(parsed, expected_values[index]); + } + + index++; + } + + ASSERT_EQUAL(index, expected_values.size()); + TEST_SUCCEED(); +} + +bool infinity_in_array() { + TEST_START(); + dom::parser parser; + auto json = R"([1, + infinity, + INFINITY, + Infinity, + inf, + Inf, + INF, + -infinity, + -INFINITY, + -Infinity, + -inf, + -Inf, + -INF, + 6.5])"_padded; + dom::element doc; + ASSERT_SUCCESS(parser.parse(json).get(doc)); + dom::array arr; + ASSERT_SUCCESS(doc.get_array().get(arr)); + + double inf = std::numeric_limits::infinity(); + std::array expected_values{ + 1, inf, inf, inf, inf, inf, inf, -inf, -inf, -inf, -inf, -inf, -inf, 6.5}; + + size_t index = 0; + for (auto val : arr) { + if (index == expected_values.size()) { + TEST_FAIL("Array contained more values than expected"); + } + + double parsed; + ASSERT_SUCCESS(val.get_double().get(parsed)); + ASSERT_EQUAL(parsed, expected_values[index]); + index++; + } + + ASSERT_EQUAL(index, expected_values.size()); + TEST_SUCCEED(); +} + +bool nan_in_object() { + TEST_START(); + dom::parser parser; + dom::element doc; + ASSERT_SUCCESS(parser.parse(R"({"a": NaN, "b": nan})"_padded).get(doc)); + double a; + ASSERT_SUCCESS(doc["a"].get_double().get(a)); + ASSERT_TRUE(std::isnan(a)); + double b; + ASSERT_SUCCESS(doc["b"].get_double().get(b)); + ASSERT_TRUE(std::isnan(b)); + TEST_SUCCEED(); +} + +bool infinity_in_object() { + TEST_START(); + dom::parser parser; + dom::element doc; + ASSERT_SUCCESS(parser.parse(R"({"a": Infinity, "b": -inf})"_padded).get(doc)); + double a; + ASSERT_SUCCESS(doc["a"].get_double().get(a)); + ASSERT_TRUE(std::isinf(a)); + ASSERT_TRUE(a > 0); + double b; + ASSERT_SUCCESS(doc["b"].get_double().get(b)); + ASSERT_TRUE(std::isinf(b)); + ASSERT_TRUE(b < 0); + TEST_SUCCEED(); +} + +// Bad 'Infinity' atoms should yield TAPE_ERROR (extension, not canonical). +// Bad 'NaN' atoms (capital 'N') should yield TAPE_ERROR. +// Bad 'nan' atoms (lowercase 'n') should yield N_ATOM_ERROR (shares the +// 'null' dispatch, so a bad 'nan' reports the same error as bad 'null'). +// Bad negative atoms (any case) should yield NUMBER_ERROR: a leading '-' +// routes through parse_number, which falls back to compute_nan_inf and +// returns NUMBER_ERROR if that fails. +// +// Each reject_* test runs the cases in three contexts: at the document root +// (visit_root_primitive), inside an array, and inside an object +// (visit_primitive). Both dispatch paths must agree on error codes. + +padded_string wrap_in_array(const char *atom) { + return padded_string(std::string("[") + atom + "]"); +} + +padded_string wrap_in_object(const char *atom) { + return padded_string(std::string("{\"key\": ") + atom + "}"); +} + +bool reject_trailing_junk() { + TEST_START(); + struct { + const char *json; + error_code expected; + } cases[] = { + {"NaNa", TAPE_ERROR}, {"NaN1", TAPE_ERROR}, + {"nana", N_ATOM_ERROR}, {"InfX", TAPE_ERROR}, + {"Inf_", TAPE_ERROR}, {"Infinityy", TAPE_ERROR}, + {"InfinityX", TAPE_ERROR}, {"-NaNa", NUMBER_ERROR}, + {"-nana", NUMBER_ERROR}, {"-InfX", NUMBER_ERROR}, + {"-Infinityy", NUMBER_ERROR}, + }; + for (auto c : cases) { + dom::parser parser; + dom::element doc; + auto err = parser.parse(padded_string(c.json, strlen(c.json))).get(doc); + ASSERT_ERROR(err, c.expected); + } + for (auto c : cases) { + dom::parser parser; + dom::element doc; + auto err = parser.parse(wrap_in_array(c.json)).get(doc); + ASSERT_ERROR(err, c.expected); + } + for (auto c : cases) { + dom::parser parser; + dom::element doc; + auto err = parser.parse(wrap_in_object(c.json)).get(doc); + ASSERT_ERROR(err, c.expected); + } + TEST_SUCCEED(); +} + +bool reject_similar_prefix() { + TEST_START(); + struct { + const char *json; + error_code expected; + } cases[] = { + {"Nope", TAPE_ERROR}, {"Napalm", TAPE_ERROR}, + {"nope", N_ATOM_ERROR}, {"Infant", TAPE_ERROR}, + {"Inform", TAPE_ERROR}, {"Information", TAPE_ERROR}, + {"-Nope", NUMBER_ERROR}, {"-nope", NUMBER_ERROR}, + {"-Infant", NUMBER_ERROR}, {"-Information", NUMBER_ERROR}, + }; + for (auto c : cases) { + dom::parser parser; + dom::element doc; + auto err = parser.parse(padded_string(c.json, strlen(c.json))).get(doc); + ASSERT_ERROR(err, c.expected); + } + for (auto c : cases) { + dom::parser parser; + dom::element doc; + auto err = parser.parse(wrap_in_array(c.json)).get(doc); + ASSERT_ERROR(err, c.expected); + } + for (auto c : cases) { + dom::parser parser; + dom::element doc; + auto err = parser.parse(wrap_in_object(c.json)).get(doc); + ASSERT_ERROR(err, c.expected); + } + TEST_SUCCEED(); +} + +bool reject_truncated_atoms() { + TEST_START(); + struct { + const char *json; + error_code expected; + } cases[] = { + {"N", TAPE_ERROR}, {"Na", TAPE_ERROR}, {"na", N_ATOM_ERROR}, + {"I", TAPE_ERROR}, {"In", TAPE_ERROR}, {"Infinit", TAPE_ERROR}, + {"-N", NUMBER_ERROR}, {"-Na", NUMBER_ERROR}, {"-na", NUMBER_ERROR}, + {"-I", NUMBER_ERROR}, {"-In", NUMBER_ERROR}, {"-Infinit", NUMBER_ERROR}, + }; + for (auto c : cases) { + dom::parser parser; + dom::element doc; + auto err = parser.parse(padded_string(c.json, strlen(c.json))).get(doc); + ASSERT_ERROR(err, c.expected); + } + for (auto c : cases) { + dom::parser parser; + dom::element doc; + auto err = parser.parse(wrap_in_array(c.json)).get(doc); + ASSERT_ERROR(err, c.expected); + } + for (auto c : cases) { + dom::parser parser; + dom::element doc; + auto err = parser.parse(wrap_in_object(c.json)).get(doc); + ASSERT_ERROR(err, c.expected); + } + TEST_SUCCEED(); +} + +bool run() { + return parse_nan() // + && parse_infinity() // + && parse_negative_infinity() // + && nan_in_array() // + && infinity_in_array() // + && nan_in_object() // + && infinity_in_object() // + && reject_trailing_junk() // + && reject_similar_prefix() // + && reject_truncated_atoms() // + ; +} + +#else // !SIMDJSON_ENABLE_NAN_INF + +bool run() { + std::cout << "NaN/Infinity parsing is disabled (SIMDJSON_ENABLE_NAN_INF=0), " + "skipping tests." + << std::endl; + return true; +} + +#endif // SIMDJSON_ENABLE_NAN_INF +} // namespace nan_inf_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, nan_inf_tests::run); +} diff --git a/tests/ondemand/CMakeLists.txt b/tests/ondemand/CMakeLists.txt index afa053b34..fb9fd7eeb 100644 --- a/tests/ondemand/CMakeLists.txt +++ b/tests/ondemand/CMakeLists.txt @@ -22,6 +22,7 @@ add_cpp_test(compile_time_json_pointer_tests LABELS ondemand acceptance add_cpp_test(compile_time_no_validation_tests LABELS ondemand acceptance per_implementation) add_cpp_test(ondemand_key_string_tests LABELS ondemand acceptance per_implementation) add_cpp_test(ondemand_misc_tests LABELS ondemand acceptance per_implementation) +add_cpp_test(ondemand_nan_inf_tests LABELS ondemand acceptance per_implementation) add_cpp_test(ondemand_number_tests LABELS ondemand acceptance per_implementation) add_cpp_test(ondemand_number_in_string_tests LABELS ondemand acceptance per_implementation) add_cpp_test(ondemand_object_tests LABELS ondemand acceptance per_implementation) diff --git a/tests/ondemand/ondemand_nan_inf_tests.cpp b/tests/ondemand/ondemand_nan_inf_tests.cpp new file mode 100644 index 000000000..a077efd2e --- /dev/null +++ b/tests/ondemand/ondemand_nan_inf_tests.cpp @@ -0,0 +1,352 @@ +#include "simdjson.h" +#include "test_ondemand.h" +#include +#include +#include + +using namespace simdjson; + +namespace nan_inf_tests { +#if SIMDJSON_ENABLE_NAN_INF + +bool parse_nan() { + TEST_START(); + for (auto json_str : {"NaN", "nan", "-nan", "-NAN"}) { + ondemand::parser parser; + padded_string json(json_str, strlen(json_str)); + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + double value; + ASSERT_SUCCESS(doc.get_double().get(value)); + ASSERT_TRUE(std::isnan(value)); + } + TEST_SUCCEED(); +} + +bool parse_infinity() { + TEST_START(); + for (auto json_str : {"Infinity", "inf", "INF", "Inf"}) { + ondemand::parser parser; + padded_string json(json_str, strlen(json_str)); + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + double value; + ASSERT_SUCCESS(doc.get_double().get(value)); + ASSERT_TRUE(std::isinf(value)); + ASSERT_TRUE(value > 0); + } + TEST_SUCCEED(); +} + +bool parse_negative_infinity() { + TEST_START(); + for (auto json_str : {"-Infinity", "-inf", "-INF", "-Inf"}) { + ondemand::parser parser; + padded_string json(json_str, strlen(json_str)); + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + double value; + ASSERT_SUCCESS(doc.get_double().get(value)); + ASSERT_TRUE(std::isinf(value)); + ASSERT_TRUE(value < 0); + } + TEST_SUCCEED(); +} + +bool nan_in_array() { + TEST_START(); + ondemand::parser parser; + auto json = R"([5, NaN, NAN, nan, -nan, -NAN, -NaN, 1.25])"_padded; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ondemand::array arr; + ASSERT_SUCCESS(doc.get_array().get(arr)); + + double nan = std::numeric_limits::quiet_NaN(); + std::array expected_values{5, nan, nan, nan, nan, nan, nan, 1.25}; + + size_t index = 0; + for (auto val : arr) { + if (index == expected_values.size()) { + TEST_FAIL("Array contained more values than expected"); + } + + double parsed; + ASSERT_SUCCESS(val.get_double().get(parsed)); + double expected = expected_values[index]; + if (std::isnan(expected)) { + ASSERT_TRUE(std::isnan(parsed)) + } else { + ASSERT_EQUAL(parsed, expected_values[index]); + } + + index++; + } + + ASSERT_EQUAL(index, expected_values.size()); + TEST_SUCCEED(); +} + +bool infinity_in_array() { + TEST_START(); + ondemand::parser parser; + auto json = R"([1, + infinity, + INFINITY, + Infinity, + inf, + Inf, + INF, + -infinity, + -INFINITY, + -Infinity, + -inf, + -Inf, + -INF, + 6.5])"_padded; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ondemand::array arr; + ASSERT_SUCCESS(doc.get_array().get(arr)); + + double inf = std::numeric_limits::infinity(); + std::array expected_values{ + 1, inf, inf, inf, inf, inf, inf, -inf, -inf, -inf, -inf, -inf, -inf, 6.5}; + + size_t index = 0; + for (auto val : arr) { + if (index == expected_values.size()) { + TEST_FAIL("Array contained more values than expected"); + } + + double parsed; + ASSERT_SUCCESS(val.get_double().get(parsed)); + ASSERT_EQUAL(parsed, expected_values[index]); + index++; + } + + ASSERT_EQUAL(index, expected_values.size()); + TEST_SUCCEED(); +} + +bool nan_in_object() { + TEST_START(); + ondemand::parser parser; + auto json = R"({"a": NaN, "b": nan})"_padded; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + double a; + ASSERT_SUCCESS(doc["a"].get_double().get(a)); + ASSERT_TRUE(std::isnan(a)); + // rewind to access second field + doc.rewind(); + double b; + ASSERT_SUCCESS(doc["b"].get_double().get(b)); + ASSERT_TRUE(std::isnan(b)); + TEST_SUCCEED(); +} + +bool infinity_in_object() { + TEST_START(); + ondemand::parser parser; + auto json = R"({"a": Infinity, "b": -inf})"_padded; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + double a; + ASSERT_SUCCESS(doc["a"].get_double().get(a)); + ASSERT_TRUE(std::isinf(a)); + ASSERT_TRUE(a > 0); + doc.rewind(); + double b; + ASSERT_SUCCESS(doc["b"].get_double().get(b)); + ASSERT_TRUE(std::isinf(b)); + ASSERT_TRUE(b < 0); + TEST_SUCCEED(); +} + +bool nan_in_string() { + TEST_START(); + for (auto json_str : {R"("NaN")", R"("nan")"}) { + ondemand::parser parser; + padded_string json(json_str, strlen(json_str)); + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + double value; + ASSERT_SUCCESS(doc.get_double_in_string().get(value)); + ASSERT_TRUE(std::isnan(value)); + } + TEST_SUCCEED(); +} + +bool infinity_in_string() { + TEST_START(); + for (auto json_str : {R"("Infinity")", R"("inf")", R"("INF")", R"("Inf")"}) { + ondemand::parser parser; + padded_string json(json_str, strlen(json_str)); + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + double value; + ASSERT_SUCCESS(doc.get_double_in_string().get(value)); + ASSERT_TRUE(std::isinf(value)); + ASSERT_TRUE(value > 0); + } + TEST_SUCCEED(); +} + +bool negative_infinity_in_string() { + TEST_START(); + for (auto json_str : + {R"("-Infinity")", R"("-inf")", R"("-INF")", R"("-Inf")"}) { + ondemand::parser parser; + padded_string json(json_str, strlen(json_str)); + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + double value; + ASSERT_SUCCESS(doc.get_double_in_string().get(value)); + ASSERT_TRUE(std::isinf(value)); + ASSERT_TRUE(value < 0); + } + TEST_SUCCEED(); +} + +bool nan_inf_in_string_in_array() { + TEST_START(); + ondemand::parser parser; + auto json = R"(["NaN", "inf", "-Infinity"])"_padded; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ondemand::array arr; + ASSERT_SUCCESS(doc.get_array().get(arr)); + size_t index = 0; + for (auto val : arr) { + double parsed; + ASSERT_SUCCESS(val.get_double_in_string().get(parsed)); + if (index == 0) { + ASSERT_TRUE(std::isnan(parsed)); + } else if (index == 1) { + ASSERT_TRUE(std::isinf(parsed)); + ASSERT_TRUE(parsed > 0); + } else { + ASSERT_TRUE(std::isinf(parsed)); + ASSERT_TRUE(parsed < 0); + } + index++; + } + ASSERT_EQUAL(index, 3); + TEST_SUCCEED(); +} + +bool nan_inf_in_string_in_object() { + TEST_START(); + ondemand::parser parser; + auto json = R"({"a": "NaN", "b": "inf", "c": "-Infinity"})"_padded; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + double a; + ASSERT_SUCCESS(doc["a"].get_double_in_string().get(a)); + ASSERT_TRUE(std::isnan(a)); + doc.rewind(); + double b; + ASSERT_SUCCESS(doc["b"].get_double_in_string().get(b)); + ASSERT_TRUE(std::isinf(b)); + ASSERT_TRUE(b > 0); + doc.rewind(); + double c; + ASSERT_SUCCESS(doc["c"].get_double_in_string().get(c)); + ASSERT_TRUE(std::isinf(c)); + ASSERT_TRUE(c < 0); + TEST_SUCCEED(); +} + +// Bad atom tokens should not be parseable as doubles, and the raw JSON +// token should still be extractable via raw_json_token(). +bool reject_trailing_junk() { + TEST_START(); + for (auto atom : + {"NaNa", "NaN1", "nana", "InfX", "Inf_", "Infinityy", "InfinityX"}) { + std::string wrapped = std::string("{\"key\": ") + atom + "}"; + ondemand::parser parser; + padded_string json(wrapped); + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + auto val = doc["key"]; + double num = 0.0; + ASSERT_ERROR(val.get(num), INCORRECT_TYPE); + std::string_view str; + ASSERT_SUCCESS(val.raw_json_token().get(str)); + ASSERT_EQUAL(str, atom); + } + TEST_SUCCEED(); +} + +bool reject_similar_prefix() { + TEST_START(); + for (auto atom : + {"Nope", "Napalm", "nope", "Infant", "Inform", "Information"}) { + std::string wrapped = std::string("{\"key\": ") + atom + "}"; + ondemand::parser parser; + padded_string json(wrapped); + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + auto val = doc["key"]; + double num = 0.0; + ASSERT_ERROR(val.get(num), INCORRECT_TYPE); + std::string_view str; + ASSERT_SUCCESS(val.raw_json_token().get(str)); + ASSERT_EQUAL(str, atom); + } + TEST_SUCCEED(); +} + +bool reject_truncated_atoms() { + TEST_START(); + for (auto atom : {"N", "Na", "na", "I", "In", "Infinit"}) { + std::string wrapped = std::string("{\"key\": ") + atom + "}"; + ondemand::parser parser; + padded_string json(wrapped); + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + auto val = doc["key"]; + double num = 0.0; + ASSERT_ERROR(val.get(num), INCORRECT_TYPE); + std::string_view str; + ASSERT_SUCCESS(val.raw_json_token().get(str)); + ASSERT_EQUAL(str, atom); + } + TEST_SUCCEED(); +} + +bool run() { + return parse_nan() // + && parse_infinity() // + && parse_negative_infinity() // + && nan_in_array() // + && infinity_in_array() // + && nan_in_object() // + && infinity_in_object() // + && nan_in_string() // + && infinity_in_string() // + && negative_infinity_in_string() // + && nan_inf_in_string_in_array() // + && nan_inf_in_string_in_object() // + && reject_trailing_junk() // + && reject_similar_prefix() // + && reject_truncated_atoms() // + ; +} + +#else // !SIMDJSON_ENABLE_NAN_INF + +bool run() { + std::cout << "NaN/Infinity parsing is disabled (SIMDJSON_ENABLE_NAN_INF=0), " + "skipping tests." + << std::endl; + return true; +} + +#endif // SIMDJSON_ENABLE_NAN_INF +} // namespace nan_inf_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, nan_inf_tests::run); +} diff --git a/tests/ondemand/ondemand_number_in_string_tests.cpp b/tests/ondemand/ondemand_number_in_string_tests.cpp index 164304d6e..e33464f7c 100644 --- a/tests/ondemand/ondemand_number_in_string_tests.cpp +++ b/tests/ondemand/ondemand_number_in_string_tests.cpp @@ -1,5 +1,6 @@ #include "simdjson.h" #include "test_ondemand.h" +#include #include using namespace simdjson; @@ -335,7 +336,12 @@ namespace number_in_string_tests { double d; std::string_view view; ASSERT_SUCCESS(doc.find_field("ticker").find_field("change").get(value)); +#if SIMDJSON_ENABLE_NAN_INF + ASSERT_SUCCESS(value.get_double_in_string().get(d)); + ASSERT_EQUAL(d, std::numeric_limits::infinity()); +#else ASSERT_ERROR(value.get_double_in_string().get(d), INCORRECT_TYPE); +#endif ASSERT_SUCCESS(value.get_string().get(view)); ASSERT_EQUAL(view,"Infinity"); TEST_SUCCEED(); @@ -365,4 +371,4 @@ namespace number_in_string_tests { int main(int argc, char *argv[]) { return test_main(argc, argv, number_in_string_tests::run); -} \ No newline at end of file +} diff --git a/tests/ondemand/ondemand_unknown_tests.cpp b/tests/ondemand/ondemand_unknown_tests.cpp index f77bb2a7c..6590e7f22 100644 --- a/tests/ondemand/ondemand_unknown_tests.cpp +++ b/tests/ondemand/ondemand_unknown_tests.cpp @@ -1,6 +1,8 @@ #include "simdjson.h" #include "test_ondemand.h" +#include + using namespace simdjson; namespace unknown_tests { @@ -23,7 +25,7 @@ namespace unknown_tests { bool object_value_type() { TEST_START(); padded_string json = "{\"key\": NaN}"_padded; - ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) { + auto do_test = [&](auto doc_result) { simdjson::ondemand::object object; ASSERT_SUCCESS( doc_result.get_object().get(object) ); @@ -31,33 +33,42 @@ namespace unknown_tests { auto val = object["key"]; ASSERT_SUCCESS( val.type().get(type) ); ASSERT_EQUAL( type, simdjson::ondemand::json_type::unknown ); - double num; + double num = 0.0; +#if SIMDJSON_ENABLE_NAN_INF + ASSERT_SUCCESS( val.get(num) ); + ASSERT_TRUE( std::isnan(num) ); +#else ASSERT_EQUAL( val.get(num), simdjson::error_code::INCORRECT_TYPE); +#endif std::string_view str; ASSERT_SUCCESS( val.raw_json_token().get(str) ); ASSERT_EQUAL( str, "NaN"); return true; - })); + }; + ASSERT_TRUE(test_ondemand_doc(json, do_test)); TEST_SUCCEED(); } #if SIMDJSON_EXCEPTIONS bool object_value_type_exception() { TEST_START(); - padded_string json = "{\"key\": NaN}"_padded; + padded_string json = "{\"key\": ThisIsNotANumber}"_padded; simdjson::ondemand::parser parser; simdjson::ondemand::document doc = parser.iterate(json); simdjson::ondemand::object object = doc.get_object(); simdjson::ondemand::value val = object["key"]; simdjson::ondemand::json_type type = val.type(); ASSERT_EQUAL( type, simdjson::ondemand::json_type::unknown); + bool exception_caught = false; try { double num = val.get_double(); (void)num; } catch (const simdjson::simdjson_error& e) { + exception_caught = true; ASSERT_EQUAL( e.error(), simdjson::error_code::INCORRECT_TYPE); } + ASSERT_TRUE( exception_caught ); std::string_view str = val.raw_json_token(); - ASSERT_EQUAL( str, "NaN"); + ASSERT_EQUAL( str, "ThisIsNotANumber"); TEST_SUCCEED(); } #endif