From 376ef7e8e91f097ffec73cb2dc1b9e71dbf068de Mon Sep 17 00:00:00 2001 From: Alecto Irene Perez Date: Thu, 7 May 2026 13:11:03 -0400 Subject: [PATCH] Add support for writing NaN/Infinity if SIMDJSON_ENABLE_NAN_INF is enabled (#2711) * Add tests for dumping out NaN/Infinity to the json builder * If SIMDJSON_ENABLE_NAN_INF is set, write out NaN and Infinity * Add tests for dumping out NaN/Infinity when serializing DOM * If SIMDJSON_ENABLE_NAN_INF, use 'NaN' and 'Infinity' for nan/inf * Fix bug where 'Inf' as a root atom + spaces of padding parses incorrectly * Update FracturedJson printers so tables containing NaN/Infinity are aligned --- include/simdjson/dom/fractured_json-inl.h | 20 +- include/simdjson/dom/serialization-inl.h | 17 ++ include/simdjson/generic/atomparsing.h | 4 +- .../generic/builder/json_string_builder-inl.h | 24 +++ .../builder/builder_string_builder_tests.cpp | 182 ++++++++++++++++-- tests/dom/nan_inf_tests.cpp | 165 +++++++++++++++- 6 files changed, 388 insertions(+), 24 deletions(-) diff --git a/include/simdjson/dom/fractured_json-inl.h b/include/simdjson/dom/fractured_json-inl.h index 2ab9d36d8..19b7ad62c 100644 --- a/include/simdjson/dom/fractured_json-inl.h +++ b/include/simdjson/dom/fractured_json-inl.h @@ -227,8 +227,18 @@ inline size_t structure_analyzer::estimate_string_length(std::string_view s) con } inline size_t structure_analyzer::estimate_number_length(double d) const { - if (std::isnan(d) || std::isinf(d)) { + if (!std::isfinite(d)) { +#if SIMDJSON_ENABLE_NAN_INF + if (std::isnan(d)) { + return 3; // "NaN" + } else if (d < 0) { + return 9; // "-Infinity" + } else { + return 8; // "Infinity" + } +#else return 4; // "null" for invalid numbers +#endif } // Rough estimate: up to 17 significant digits + sign + decimal point + exponent char buf[32]; @@ -950,6 +960,14 @@ inline size_t fractured_string_builder::measure_value_length(const dom::element& case dom::element_type::DOUBLE: { double val; if (elem.get_double().get(val) == SUCCESS) { +#if SIMDJSON_ENABLE_NAN_INF + if (!std::isfinite(val)) { + if (std::isnan(val)) + return 3; // "NaN" + // "-Infinity" (9) or "Infinity" (8) + return val < 0 ? 9 : 8; + } +#endif char buf[32]; int len = snprintf(buf, sizeof(buf), "%.17g", val); return len > 0 ? static_cast(len) : 1; diff --git a/include/simdjson/dom/serialization-inl.h b/include/simdjson/dom/serialization-inl.h index cfb963cad..b58c9984f 100644 --- a/include/simdjson/dom/serialization-inl.h +++ b/include/simdjson/dom/serialization-inl.h @@ -11,6 +11,7 @@ #include "simdjson/dom/object-inl.h" #include "simdjson/internal/tape_ref-inl.h" +#include #include namespace simdjson { @@ -181,6 +182,22 @@ simdjson_inline void base_formatter::number(int64_t x) { template simdjson_inline void base_formatter::number(double x) { +#if SIMDJSON_ENABLE_NAN_INF + if (simdjson_unlikely(!std::isfinite(x))) { + if (std::isnan(x)) { + char const *s = "NaN"; + chars(s, s + 3); + } else { + if (x < 0) { + one_char('-'); + } + char const *s = "Infinity"; + chars(s, s + 8); + } + return; + } +#endif + char number_buffer[24]; // Currently, passing the nullptr to the second argument is // safe because our implementation does not check the second diff --git a/include/simdjson/generic/atomparsing.h b/include/simdjson/generic/atomparsing.h index 598c5b079..20e77dd82 100644 --- a/include/simdjson/generic/atomparsing.h +++ b/include/simdjson/generic/atomparsing.h @@ -147,7 +147,9 @@ simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { simdjson_warn_unused simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8) { return str8ncmp_case_insensitive(src, "infinity") == 0; } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } if (len > 3) { return (str3ncmp_case_insensitive(src, "inf") | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; diff --git a/include/simdjson/generic/builder/json_string_builder-inl.h b/include/simdjson/generic/builder/json_string_builder-inl.h index 5c13af13e..809babe5a 100644 --- a/include/simdjson/generic/builder/json_string_builder-inl.h +++ b/include/simdjson/generic/builder/json_string_builder-inl.h @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -710,6 +711,29 @@ simdjson_inline void string_builder::append(number_type v) noexcept { else SIMDJSON_IF_CONSTEXPR(std::is_floating_point::value) { constexpr size_t max_number_size = 24; if (capacity_check(max_number_size)) { +#if SIMDJSON_ENABLE_NAN_INF + // Check if the input might be NaN or infinity + if (simdjson_unlikely(!std::isfinite(v))) { + if (std::isnan(v)) { + constexpr char nan_literal[] = "NaN"; + constexpr size_t nan_len = sizeof(nan_literal) - 1; + + std::memcpy(buffer.get() + position, nan_literal, nan_len); + position += nan_len; + } else { + constexpr char inf_literal[] = "Infinity"; + constexpr size_t inf_len = sizeof(inf_literal) - 1; + if (v < 0) { + buffer.get()[position] = '-'; + ++position; + } + std::memcpy(buffer.get() + position, inf_literal, inf_len); + position += inf_len; + } + return; + } +#endif + // We could specialize for float. char *end = simdjson::internal::to_chars(buffer.get() + position, nullptr, double(v)); diff --git a/tests/builder/builder_string_builder_tests.cpp b/tests/builder/builder_string_builder_tests.cpp index c67fab13f..bc99a2064 100644 --- a/tests/builder/builder_string_builder_tests.cpp +++ b/tests/builder/builder_string_builder_tests.cpp @@ -1,5 +1,8 @@ #include "simdjson.h" #include "test_builder.h" +#include +#include +#include #include #include #include @@ -17,28 +20,27 @@ struct Car { std::vector tire_pressure; }; // Car - #if SIMDJSON_SUPPORTS_CONCEPTS struct Car2549 { - std::string make; - std::string model; - int64_t year; - std::vector tire_pressure; + std::string make; + std::string model; + int64_t year; + std::vector tire_pressure; }; namespace simdjson { - // we intentionally pass by non-const reference to car. - template - void tag_invoke(serialize_tag, builder_type& builder, Car2549& car) { - builder.start_object(); - builder.append_key_value("make", car.make); - builder.append_comma(); - builder.append_key_value("model", car.model); - builder.append_comma(); - builder.append_key_value("year", car.year); - builder.append_comma(); - builder.append_key_value("tire_pressure", car.tire_pressure); - builder.end_object(); - } +// we intentionally pass by non-const reference to car. +template +void tag_invoke(serialize_tag, builder_type &builder, Car2549 &car) { + builder.start_object(); + builder.append_key_value("make", car.make); + builder.append_comma(); + builder.append_key_value("model", car.model); + builder.append_comma(); + builder.append_key_value("year", car.year); + builder.append_comma(); + builder.append_key_value("tire_pressure", car.tire_pressure); + builder.end_object(); +} } // namespace simdjson static_assert(simdjson::require_custom_serialization); @@ -160,6 +162,140 @@ bool append_float() { TEST_SUCCEED(); } +#if SIMDJSON_ENABLE_NAN_INF +bool append_nan() { + TEST_START(); + simdjson::builder::string_builder sb; + sb.append(std::numeric_limits::quiet_NaN()); + std::string_view p; + ASSERT_SUCCESS(sb.view().get(p)); + ASSERT_EQUAL(p, "NaN"); + TEST_SUCCEED(); +} + +bool append_positive_infinity() { + TEST_START(); + simdjson::builder::string_builder sb; + sb.append(std::numeric_limits::infinity()); + std::string_view p; + ASSERT_SUCCESS(sb.view().get(p)); + ASSERT_EQUAL(p, "Infinity"); + TEST_SUCCEED(); +} + +bool append_negative_infinity() { + TEST_START(); + simdjson::builder::string_builder sb; + sb.append(-std::numeric_limits::infinity()); + std::string_view p; + ASSERT_SUCCESS(sb.view().get(p)); + ASSERT_EQUAL(p, "-Infinity"); + TEST_SUCCEED(); +} + +bool append_float_nan_inf() { + TEST_START(); + { + simdjson::builder::string_builder sb; + sb.append(std::numeric_limits::quiet_NaN()); + std::string_view p; + ASSERT_SUCCESS(sb.view().get(p)); + ASSERT_EQUAL(p, "NaN"); + } + { + simdjson::builder::string_builder sb; + sb.append(std::numeric_limits::infinity()); + std::string_view p; + ASSERT_SUCCESS(sb.view().get(p)); + ASSERT_EQUAL(p, "Infinity"); + } + { + simdjson::builder::string_builder sb; + sb.append(-std::numeric_limits::infinity()); + std::string_view p; + ASSERT_SUCCESS(sb.view().get(p)); + ASSERT_EQUAL(p, "-Infinity"); + } + TEST_SUCCEED(); +} + +bool nan_inf_in_array() { + TEST_START(); + simdjson::builder::string_builder sb; + sb.start_array(); + sb.append(1.5); + sb.append_comma(); + sb.append(std::numeric_limits::quiet_NaN()); + sb.append_comma(); + sb.append(std::numeric_limits::infinity()); + sb.append_comma(); + sb.append(-std::numeric_limits::infinity()); + sb.append_comma(); + sb.append(2.5); + sb.end_array(); + std::string_view p; + ASSERT_SUCCESS(sb.view().get(p)); + ASSERT_EQUAL(p, "[1.5,NaN,Infinity,-Infinity,2.5]"); + TEST_SUCCEED(); +} + +bool nan_inf_in_object() { + TEST_START(); + simdjson::builder::string_builder sb; + sb.start_object(); + sb.append_key_value("a", std::numeric_limits::quiet_NaN()); + sb.append_comma(); + sb.append_key_value("b", std::numeric_limits::infinity()); + sb.append_comma(); + sb.append_key_value("c", -std::numeric_limits::infinity()); + sb.end_object(); + std::string_view p; + ASSERT_SUCCESS(sb.view().get(p)); + ASSERT_EQUAL(p, "{\"a\":NaN,\"b\":Infinity,\"c\":-Infinity}"); + TEST_SUCCEED(); +} + +bool nan_inf_roundtrip() { + TEST_START(); + simdjson::builder::string_builder sb; + sb.start_array(); + sb.append(std::numeric_limits::quiet_NaN()); + sb.append_comma(); + sb.append(std::numeric_limits::infinity()); + sb.append_comma(); + sb.append(-std::numeric_limits::infinity()); + sb.end_array(); + std::string_view p; + ASSERT_SUCCESS(sb.view().get(p)); + + simdjson::padded_string output{p}; + simdjson::dom::parser parser; + simdjson::dom::element doc; + ASSERT_SUCCESS(parser.parse(output).get(doc)); + simdjson::dom::array arr; + ASSERT_SUCCESS(doc.get_array().get(arr)); + + std::array expected{ + std::numeric_limits::quiet_NaN(), + std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), + }; + size_t index = 0; + for (auto val : arr) { + double parsed; + ASSERT_SUCCESS(val.get_double().get(parsed)); + if (std::isnan(expected[index])) { + ASSERT_TRUE(std::isnan(parsed)); + } else { + ASSERT_EQUAL(parsed, expected[index]); + } + index++; + } + ASSERT_EQUAL(index, expected.size()); + TEST_SUCCEED(); +} +#endif // SIMDJSON_ENABLE_NAN_INF + bool append_null() { TEST_START(); simdjson::builder::string_builder sb; @@ -459,14 +595,15 @@ bool car_test() { bool issue2549() { TEST_START(); simdjson::builder::string_builder sb; - Car2549 c = { "Toyota", "Corolla", 2017, {1.0f,2.0f,3.0f} }; + Car2549 c = {"Toyota", "Corolla", 2017, {1.0f, 2.0f, 3.0f}}; sb.start_object(); sb.append_key_value("car", c); sb.end_object(); std::string_view p; auto result = sb.view().get(p); ASSERT_SUCCESS(result); - ASSERT_EQUAL(p, "{\"car\":{\"make\":\"Toyota\",\"model\":\"Corolla\",\"year\":2017,\"tire_pressure\":[1.0,2.0,3.0]}}"); + ASSERT_EQUAL(p, "{\"car\":{\"make\":\"Toyota\",\"model\":\"Corolla\"," + "\"year\":2017,\"tire_pressure\":[1.0,2.0,3.0]}}"); TEST_SUCCEED(); } @@ -661,6 +798,11 @@ bool run() { issue2549() && car_test_template() && serialize_optional() && #endif append_char() && append_integer() && append_float() && append_null() && +#if SIMDJSON_ENABLE_NAN_INF + append_nan() && append_positive_infinity() && + append_negative_infinity() && append_float_nan_inf() && + nan_inf_in_array() && nan_inf_in_object() && nan_inf_roundtrip() && +#endif clear() && escape_and_append() && escape_and_append_with_quotes() && append_raw() && raw_with_length() && string_convertion() && buffer_growth() && unicode_validation() && true; diff --git a/tests/dom/nan_inf_tests.cpp b/tests/dom/nan_inf_tests.cpp index 038e31a7f..e62c18276 100644 --- a/tests/dom/nan_inf_tests.cpp +++ b/tests/dom/nan_inf_tests.cpp @@ -1,7 +1,9 @@ #include "simdjson.h" #include "test_macros.h" #include "test_main.h" +#include #include +#include #include using namespace simdjson; @@ -25,8 +27,10 @@ bool parse_nan() { bool parse_infinity() { TEST_START(); - for (auto json_str : - {"infinity", "Infinity", "INFINITY", "inf", "Inf", "INF"}) { + for (auto json_str : {"infinity", "Infinity", "INFINITY", "inf", "Inf", "INF", + // Check that 'Inf' parses correctly even when padded to + // the same length as 'Infinity' + "inf ", "Inf ", "INF "}) { dom::parser parser; dom::element doc; ASSERT_SUCCESS( @@ -280,6 +284,154 @@ bool reject_truncated_atoms() { TEST_SUCCEED(); } +// DOM printer (to_string / minify / prettify) tests. When NaN/Infinity +// parsing is enabled, the writer must emit the same literals on output so +// that round-tripping through the parser preserves the value. + +bool print_nan() { + TEST_START(); + dom::parser parser; + dom::element doc; + ASSERT_SUCCESS(parser.parse("NaN"_padded).get(doc)); + ASSERT_EQUAL(simdjson::to_string(doc), "NaN"); + ASSERT_EQUAL(simdjson::minify(doc), "NaN"); + TEST_SUCCEED(); +} + +bool print_infinity() { + TEST_START(); + dom::parser parser; + dom::element doc; + ASSERT_SUCCESS(parser.parse("Infinity"_padded).get(doc)); + ASSERT_EQUAL(simdjson::to_string(doc), "Infinity"); + ASSERT_EQUAL(simdjson::minify(doc), "Infinity"); + TEST_SUCCEED(); +} + +bool print_negative_infinity() { + TEST_START(); + dom::parser parser; + dom::element doc; + ASSERT_SUCCESS(parser.parse("-Infinity"_padded).get(doc)); + ASSERT_EQUAL(simdjson::to_string(doc), "-Infinity"); + ASSERT_EQUAL(simdjson::minify(doc), "-Infinity"); + TEST_SUCCEED(); +} + +bool print_nan_in_array() { + TEST_START(); + dom::parser parser; + dom::element doc; + ASSERT_SUCCESS( + parser.parse("[1.5, NaN, Infinity, -Infinity, 2.5]"_padded).get(doc)); + ASSERT_EQUAL(simdjson::to_string(doc), "[1.5,NaN,Infinity,-Infinity,2.5]"); + TEST_SUCCEED(); +} + +bool print_nan_in_object() { + TEST_START(); + dom::parser parser; + dom::element doc; + ASSERT_SUCCESS( + parser.parse(R"({"a": NaN, "b": Infinity, "c": -Infinity})"_padded) + .get(doc)); + ASSERT_EQUAL(simdjson::to_string(doc), + "{\"a\":NaN,\"b\":Infinity,\"c\":-Infinity}"); + TEST_SUCCEED(); +} + +bool print_roundtrip() { + TEST_START(); + dom::parser parser; + dom::element doc; + ASSERT_SUCCESS(parser.parse("[NaN, Infinity, -Infinity]"_padded).get(doc)); + std::string serialized = simdjson::to_string(doc); + + dom::parser parser2; + dom::element doc2; + ASSERT_SUCCESS(parser2.parse(padded_string(serialized)).get(doc2)); + dom::array arr; + ASSERT_SUCCESS(doc2.get_array().get(arr)); + + std::array expected{ + std::numeric_limits::quiet_NaN(), + std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), + }; + size_t index = 0; + for (auto val : arr) { + double parsed; + ASSERT_SUCCESS(val.get_double().get(parsed)); + if (std::isnan(expected[index])) { + ASSERT_TRUE(std::isnan(parsed)); + } else { + ASSERT_EQUAL(parsed, expected[index]); + } + index++; + } + ASSERT_EQUAL(index, expected.size()); + TEST_SUCCEED(); +} + +// FracturedJson aligns values into columns in table mode. The column width +// is driven by the estimator for unseen elements and by measure_value_length +// for the chosen cells; if either undercounts NaN/Infinity, column 1's +// padding won't match column 2's emitted width and rows visibly misalign. +// Force table mode with min_table_rows = 2 and max_inline_length = 0. + +bool table_aligns_nan() { + TEST_START(); + dom::parser parser; + dom::element doc; + ASSERT_SUCCESS( + parser.parse(R"([{"a": 1, "b": 1},{"a": NaN, "b": 1}])"_padded).get(doc)); + fractured_json_options opts; + opts.min_table_rows = 2; + opts.max_inline_length = 0; + ASSERT_EQUAL(simdjson::fractured_json(doc, opts), + "[\n" + " { \"a\": 1 , \"b\": 1 },\n" + " { \"a\": NaN, \"b\": 1 }\n" + "]"); + TEST_SUCCEED(); +} + +bool table_aligns_inf() { + TEST_START(); + dom::parser parser; + dom::element doc; + ASSERT_SUCCESS( + parser.parse(R"([{"a": 1, "b": 1},{"a": Infinity, "b": 1}])"_padded) + .get(doc)); + fractured_json_options opts; + opts.min_table_rows = 2; + opts.max_inline_length = 0; + ASSERT_EQUAL(simdjson::fractured_json(doc, opts), + "[\n" + " { \"a\": 1 , \"b\": 1 },\n" + " { \"a\": Infinity, \"b\": 1 }\n" + "]"); + TEST_SUCCEED(); +} + +bool table_aligns_neg_inf() { + TEST_START(); + dom::parser parser; + dom::element doc; + ASSERT_SUCCESS( + parser.parse(R"([{"a": 1, "b": 1},{"a": -Infinity, "b": 1}])"_padded) + .get(doc)); + fractured_json_options opts; + opts.min_table_rows = 2; + opts.max_inline_length = 0; + ASSERT_EQUAL(simdjson::fractured_json(doc, opts), + "[\n" + " { \"a\": 1 , \"b\": 1 },\n" + " { \"a\": -Infinity, \"b\": 1 }\n" + "]"); + TEST_SUCCEED(); +} + bool run() { return parse_nan() // && parse_infinity() // @@ -291,6 +443,15 @@ bool run() { && reject_trailing_junk() // && reject_similar_prefix() // && reject_truncated_atoms() // + && print_nan() // + && print_infinity() // + && print_negative_infinity() // + && print_nan_in_array() // + && print_nan_in_object() // + && print_roundtrip() // + && table_aligns_nan() // + && table_aligns_inf() // + && table_aligns_neg_inf() // ; }