admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support for writing NaN/Infinity if SIMDJSON_ENABLE_NAN_INF is enabled (#2711)

Browse Source 
* Add tests for dumping out NaN/Infinity to the json builder

* If SIMDJSON_ENABLE_NAN_INF is set, write out NaN and Infinity

* Add tests for dumping out NaN/Infinity when serializing DOM

* If SIMDJSON_ENABLE_NAN_INF, use 'NaN' and 'Infinity' for nan/inf

* Fix bug where 'Inf' as a root atom + spaces of padding parses incorrectly

* Update FracturedJson printers so tables containing NaN/Infinity are aligned
This commit is contained in:
Alecto Irene Perez
2026-05-07 13:11:03 -04:00
committed by GitHub
parent 0a23ebbfef
commit 376ef7e8e9
6 changed files with 388 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/simdjson/dom/fractured_json-inl.h
+19 -1
View File
@@ -227,8 +227,18 @@ inline size_t structure_analyzer::estimate_string_length(std::string_view s) con
}
inline size_t structure_analyzer::estimate_number_length(double d) const {
if (std::isnan(d) || std::isinf(d)) {
if (!std::isfinite(d)) {
#if SIMDJSON_ENABLE_NAN_INF
if (std::isnan(d)) {
return 3; // "NaN"
} else if (d < 0) {
return 9; // "-Infinity"
} else {
return 8; // "Infinity"
}
#else
return 4; // "null" for invalid numbers
#endif
}
// Rough estimate: up to 17 significant digits + sign + decimal point + exponent
char buf[32];
@@ -950,6 +960,14 @@ inline size_t fractured_string_builder::measure_value_length(const dom::element&
case dom::element_type::DOUBLE: {
double val;
if (elem.get_double().get(val) == SUCCESS) {
#if SIMDJSON_ENABLE_NAN_INF
if (!std::isfinite(val)) {
if (std::isnan(val))
return 3; // "NaN"
// "-Infinity" (9) or "Infinity" (8)
return val < 0 ? 9 : 8;
}
#endif
char buf[32];
int len = snprintf(buf, sizeof(buf), "%.17g", val);
return len > 0 ? static_cast<size_t>(len) : 1;
include/simdjson/dom/serialization-inl.h
+17
View File
@@ -11,6 +11,7 @@
#include "simdjson/dom/object-inl.h"
#include "simdjson/internal/tape_ref-inl.h"
#include <cmath>
#include <cstring>
namespace simdjson {
@@ -181,6 +182,22 @@ simdjson_inline void base_formatter<formatter>::number(int64_t x) {
template <class formatter>
simdjson_inline void base_formatter<formatter>::number(double x) {
#if SIMDJSON_ENABLE_NAN_INF
if (simdjson_unlikely(!std::isfinite(x))) {
if (std::isnan(x)) {
char const *s = "NaN";
chars(s, s + 3);
} else {
if (x < 0) {
one_char('-');
}
char const *s = "Infinity";
chars(s, s + 8);
}
return;
}
#endif
char number_buffer[24];
// Currently, passing the nullptr to the second argument is
// safe because our implementation does not check the second
include/simdjson/generic/atomparsing.h
+3 -1
View File
@@ -147,7 +147,9 @@ simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) {
simdjson_warn_unused
simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) {
if (len > 8) { return is_valid_inf_atom(src); }
if (len == 8) { return str8ncmp_case_insensitive(src, "infinity") == 0; }
if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) {
return true;
}
if (len > 3) {
return (str3ncmp_case_insensitive(src, "inf")
| jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0;
include/simdjson/generic/builder/json_string_builder-inl.h
+24
View File
@@ -1,4 +1,5 @@
#include <array>
#include <cmath>
#include <cstring>
#include <limits>
#include <type_traits>
@@ -710,6 +711,29 @@ simdjson_inline void string_builder::append(number_type v) noexcept {
else SIMDJSON_IF_CONSTEXPR(std::is_floating_point<number_type>::value) {
constexpr size_t max_number_size = 24;
if (capacity_check(max_number_size)) {
#if SIMDJSON_ENABLE_NAN_INF
// Check if the input might be NaN or infinity
if (simdjson_unlikely(!std::isfinite(v))) {
if (std::isnan(v)) {
constexpr char nan_literal[] = "NaN";
constexpr size_t nan_len = sizeof(nan_literal) - 1;
std::memcpy(buffer.get() + position, nan_literal, nan_len);
position += nan_len;
} else {
constexpr char inf_literal[] = "Infinity";
constexpr size_t inf_len = sizeof(inf_literal) - 1;
if (v < 0) {
buffer.get()[position] = '-';
++position;
}
std::memcpy(buffer.get() + position, inf_literal, inf_len);
position += inf_len;
}
return;
}
#endif
// We could specialize for float.
char *end = simdjson::internal::to_chars(buffer.get() + position, nullptr,
double(v));
tests/builder/builder_string_builder_tests.cpp
+162 -20
View File
@@ -1,5 +1,8 @@
#include "simdjson.h"
#include "test_builder.h"
#include <array>
#include <cmath>
#include <limits>
#include <map>
#include <string>
#include <string_view>
@@ -17,28 +20,27 @@ struct Car {
std::vector<double> tire_pressure;
}; // Car
#if SIMDJSON_SUPPORTS_CONCEPTS
struct Car2549 {
std::string make;
std::string model;
int64_t year;
std::vector<float> tire_pressure;
std::string make;
std::string model;
int64_t year;
std::vector<float> tire_pressure;
};
namespace simdjson {
// we intentionally pass by non-const reference to car.
template <typename builder_type>
void tag_invoke(serialize_tag, builder_type& builder, Car2549& car) {
builder.start_object();
builder.append_key_value("make", car.make);
builder.append_comma();
builder.append_key_value("model", car.model);
builder.append_comma();
builder.append_key_value("year", car.year);
builder.append_comma();
builder.append_key_value("tire_pressure", car.tire_pressure);
builder.end_object();
}
// we intentionally pass by non-const reference to car.
template <typename builder_type>
void tag_invoke(serialize_tag, builder_type &builder, Car2549 &car) {
builder.start_object();
builder.append_key_value("make", car.make);
builder.append_comma();
builder.append_key_value("model", car.model);
builder.append_comma();
builder.append_key_value("year", car.year);
builder.append_comma();
builder.append_key_value("tire_pressure", car.tire_pressure);
builder.end_object();
}
} // namespace simdjson
static_assert(simdjson::require_custom_serialization<Car2549>);
@@ -160,6 +162,140 @@ bool append_float() {
TEST_SUCCEED();
}
#if SIMDJSON_ENABLE_NAN_INF
bool append_nan() {
TEST_START();
simdjson::builder::string_builder sb;
sb.append(std::numeric_limits<double>::quiet_NaN());
std::string_view p;
ASSERT_SUCCESS(sb.view().get(p));
ASSERT_EQUAL(p, "NaN");
TEST_SUCCEED();
}
bool append_positive_infinity() {
TEST_START();
simdjson::builder::string_builder sb;
sb.append(std::numeric_limits<double>::infinity());
std::string_view p;
ASSERT_SUCCESS(sb.view().get(p));
ASSERT_EQUAL(p, "Infinity");
TEST_SUCCEED();
}
bool append_negative_infinity() {
TEST_START();
simdjson::builder::string_builder sb;
sb.append(-std::numeric_limits<double>::infinity());
std::string_view p;
ASSERT_SUCCESS(sb.view().get(p));
ASSERT_EQUAL(p, "-Infinity");
TEST_SUCCEED();
}
bool append_float_nan_inf() {
TEST_START();
{
simdjson::builder::string_builder sb;
sb.append(std::numeric_limits<float>::quiet_NaN());
std::string_view p;
ASSERT_SUCCESS(sb.view().get(p));
ASSERT_EQUAL(p, "NaN");
}
{
simdjson::builder::string_builder sb;
sb.append(std::numeric_limits<float>::infinity());
std::string_view p;
ASSERT_SUCCESS(sb.view().get(p));
ASSERT_EQUAL(p, "Infinity");
}
{
simdjson::builder::string_builder sb;
sb.append(-std::numeric_limits<float>::infinity());
std::string_view p;
ASSERT_SUCCESS(sb.view().get(p));
ASSERT_EQUAL(p, "-Infinity");
}
TEST_SUCCEED();
}
bool nan_inf_in_array() {
TEST_START();
simdjson::builder::string_builder sb;
sb.start_array();
sb.append(1.5);
sb.append_comma();
sb.append(std::numeric_limits<double>::quiet_NaN());
sb.append_comma();
sb.append(std::numeric_limits<double>::infinity());
sb.append_comma();
sb.append(-std::numeric_limits<double>::infinity());
sb.append_comma();
sb.append(2.5);
sb.end_array();
std::string_view p;
ASSERT_SUCCESS(sb.view().get(p));
ASSERT_EQUAL(p, "[1.5,NaN,Infinity,-Infinity,2.5]");
TEST_SUCCEED();
}
bool nan_inf_in_object() {
TEST_START();
simdjson::builder::string_builder sb;
sb.start_object();
sb.append_key_value("a", std::numeric_limits<double>::quiet_NaN());
sb.append_comma();
sb.append_key_value("b", std::numeric_limits<double>::infinity());
sb.append_comma();
sb.append_key_value("c", -std::numeric_limits<double>::infinity());
sb.end_object();
std::string_view p;
ASSERT_SUCCESS(sb.view().get(p));
ASSERT_EQUAL(p, "{\"a\":NaN,\"b\":Infinity,\"c\":-Infinity}");
TEST_SUCCEED();
}
bool nan_inf_roundtrip() {
TEST_START();
simdjson::builder::string_builder sb;
sb.start_array();
sb.append(std::numeric_limits<double>::quiet_NaN());
sb.append_comma();
sb.append(std::numeric_limits<double>::infinity());
sb.append_comma();
sb.append(-std::numeric_limits<double>::infinity());
sb.end_array();
std::string_view p;
ASSERT_SUCCESS(sb.view().get(p));
simdjson::padded_string output{p};
simdjson::dom::parser parser;
simdjson::dom::element doc;
ASSERT_SUCCESS(parser.parse(output).get(doc));
simdjson::dom::array arr;
ASSERT_SUCCESS(doc.get_array().get(arr));
std::array<double, 3> expected{
std::numeric_limits<double>::quiet_NaN(),
std::numeric_limits<double>::infinity(),
-std::numeric_limits<double>::infinity(),
};
size_t index = 0;
for (auto val : arr) {
double parsed;
ASSERT_SUCCESS(val.get_double().get(parsed));
if (std::isnan(expected[index])) {
ASSERT_TRUE(std::isnan(parsed));
} else {
ASSERT_EQUAL(parsed, expected[index]);
}
index++;
}
ASSERT_EQUAL(index, expected.size());
TEST_SUCCEED();
}
#endif // SIMDJSON_ENABLE_NAN_INF
bool append_null() {
TEST_START();
simdjson::builder::string_builder sb;
@@ -459,14 +595,15 @@ bool car_test() {
bool issue2549() {
TEST_START();
simdjson::builder::string_builder sb;
Car2549 c = { "Toyota", "Corolla", 2017, {1.0f,2.0f,3.0f} };
Car2549 c = {"Toyota", "Corolla", 2017, {1.0f, 2.0f, 3.0f}};
sb.start_object();
sb.append_key_value("car", c);
sb.end_object();
std::string_view p;
auto result = sb.view().get(p);
ASSERT_SUCCESS(result);
ASSERT_EQUAL(p, "{\"car\":{\"make\":\"Toyota\",\"model\":\"Corolla\",\"year\":2017,\"tire_pressure\":[1.0,2.0,3.0]}}");
ASSERT_EQUAL(p, "{\"car\":{\"make\":\"Toyota\",\"model\":\"Corolla\","
"\"year\":2017,\"tire_pressure\":[1.0,2.0,3.0]}}");
TEST_SUCCEED();
}
@@ -661,6 +798,11 @@ bool run() {
issue2549() && car_test_template() && serialize_optional() &&
#endif
append_char() && append_integer() && append_float() && append_null() &&
#if SIMDJSON_ENABLE_NAN_INF
append_nan() && append_positive_infinity() &&
append_negative_infinity() && append_float_nan_inf() &&
nan_inf_in_array() && nan_inf_in_object() && nan_inf_roundtrip() &&
#endif
clear() && escape_and_append() && escape_and_append_with_quotes() &&
append_raw() && raw_with_length() && string_convertion() &&
buffer_growth() && unicode_validation() && true;
tests/dom/nan_inf_tests.cpp
+163 -2
View File
@@ -1,7 +1,9 @@
#include "simdjson.h"
#include "test_macros.h"
#include "test_main.h"
#include <array>
#include <cmath>
#include <limits>
#include <string>
using namespace simdjson;
@@ -25,8 +27,10 @@ bool parse_nan() {
bool parse_infinity() {
TEST_START();
for (auto json_str :
{"infinity", "Infinity", "INFINITY", "inf", "Inf", "INF"}) {
for (auto json_str : {"infinity", "Infinity", "INFINITY", "inf", "Inf", "INF",
// Check that 'Inf' parses correctly even when padded to
// the same length as 'Infinity'
"inf ", "Inf ", "INF "}) {
dom::parser parser;
dom::element doc;
ASSERT_SUCCESS(
@@ -280,6 +284,154 @@ bool reject_truncated_atoms() {
TEST_SUCCEED();
}
// DOM printer (to_string / minify / prettify) tests. When NaN/Infinity
// parsing is enabled, the writer must emit the same literals on output so
// that round-tripping through the parser preserves the value.
bool print_nan() {
TEST_START();
dom::parser parser;
dom::element doc;
ASSERT_SUCCESS(parser.parse("NaN"_padded).get(doc));
ASSERT_EQUAL(simdjson::to_string(doc), "NaN");
ASSERT_EQUAL(simdjson::minify(doc), "NaN");
TEST_SUCCEED();
}
bool print_infinity() {
TEST_START();
dom::parser parser;
dom::element doc;
ASSERT_SUCCESS(parser.parse("Infinity"_padded).get(doc));
ASSERT_EQUAL(simdjson::to_string(doc), "Infinity");
ASSERT_EQUAL(simdjson::minify(doc), "Infinity");
TEST_SUCCEED();
}
bool print_negative_infinity() {
TEST_START();
dom::parser parser;
dom::element doc;
ASSERT_SUCCESS(parser.parse("-Infinity"_padded).get(doc));
ASSERT_EQUAL(simdjson::to_string(doc), "-Infinity");
ASSERT_EQUAL(simdjson::minify(doc), "-Infinity");
TEST_SUCCEED();
}
bool print_nan_in_array() {
TEST_START();
dom::parser parser;
dom::element doc;
ASSERT_SUCCESS(
parser.parse("[1.5, NaN, Infinity, -Infinity, 2.5]"_padded).get(doc));
ASSERT_EQUAL(simdjson::to_string(doc), "[1.5,NaN,Infinity,-Infinity,2.5]");
TEST_SUCCEED();
}
bool print_nan_in_object() {
TEST_START();
dom::parser parser;
dom::element doc;
ASSERT_SUCCESS(
parser.parse(R"({"a": NaN, "b": Infinity, "c": -Infinity})"_padded)
.get(doc));
ASSERT_EQUAL(simdjson::to_string(doc),
"{\"a\":NaN,\"b\":Infinity,\"c\":-Infinity}");
TEST_SUCCEED();
}
bool print_roundtrip() {
TEST_START();
dom::parser parser;
dom::element doc;
ASSERT_SUCCESS(parser.parse("[NaN, Infinity, -Infinity]"_padded).get(doc));
std::string serialized = simdjson::to_string(doc);
dom::parser parser2;
dom::element doc2;
ASSERT_SUCCESS(parser2.parse(padded_string(serialized)).get(doc2));
dom::array arr;
ASSERT_SUCCESS(doc2.get_array().get(arr));
std::array<double, 3> expected{
std::numeric_limits<double>::quiet_NaN(),
std::numeric_limits<double>::infinity(),
-std::numeric_limits<double>::infinity(),
};
size_t index = 0;
for (auto val : arr) {
double parsed;
ASSERT_SUCCESS(val.get_double().get(parsed));
if (std::isnan(expected[index])) {
ASSERT_TRUE(std::isnan(parsed));
} else {
ASSERT_EQUAL(parsed, expected[index]);
}
index++;
}
ASSERT_EQUAL(index, expected.size());
TEST_SUCCEED();
}
// FracturedJson aligns values into columns in table mode. The column width
// is driven by the estimator for unseen elements and by measure_value_length
// for the chosen cells; if either undercounts NaN/Infinity, column 1's
// padding won't match column 2's emitted width and rows visibly misalign.
// Force table mode with min_table_rows = 2 and max_inline_length = 0.
bool table_aligns_nan() {
TEST_START();
dom::parser parser;
dom::element doc;
ASSERT_SUCCESS(
parser.parse(R"([{"a": 1, "b": 1},{"a": NaN, "b": 1}])"_padded).get(doc));
fractured_json_options opts;
opts.min_table_rows = 2;
opts.max_inline_length = 0;
ASSERT_EQUAL(simdjson::fractured_json(doc, opts),
"[\n"
" { \"a\": 1 , \"b\": 1 },\n"
" { \"a\": NaN, \"b\": 1 }\n"
"]");
TEST_SUCCEED();
}
bool table_aligns_inf() {
TEST_START();
dom::parser parser;
dom::element doc;
ASSERT_SUCCESS(
parser.parse(R"([{"a": 1, "b": 1},{"a": Infinity, "b": 1}])"_padded)
.get(doc));
fractured_json_options opts;
opts.min_table_rows = 2;
opts.max_inline_length = 0;
ASSERT_EQUAL(simdjson::fractured_json(doc, opts),
"[\n"
" { \"a\": 1 , \"b\": 1 },\n"
" { \"a\": Infinity, \"b\": 1 }\n"
"]");
TEST_SUCCEED();
}
bool table_aligns_neg_inf() {
TEST_START();
dom::parser parser;
dom::element doc;
ASSERT_SUCCESS(
parser.parse(R"([{"a": 1, "b": 1},{"a": -Infinity, "b": 1}])"_padded)
.get(doc));
fractured_json_options opts;
opts.min_table_rows = 2;
opts.max_inline_length = 0;
ASSERT_EQUAL(simdjson::fractured_json(doc, opts),
"[\n"
" { \"a\": 1 , \"b\": 1 },\n"
" { \"a\": -Infinity, \"b\": 1 }\n"
"]");
TEST_SUCCEED();
}
bool run() {
return parse_nan() //
&& parse_infinity() //
@@ -291,6 +443,15 @@ bool run() {
&& reject_trailing_junk() //
&& reject_similar_prefix() //
&& reject_truncated_atoms() //
&& print_nan() //
&& print_infinity() //
&& print_negative_infinity() //
&& print_nan_in_array() //
&& print_nan_in_object() //
&& print_roundtrip() //
&& table_aligns_nan() //
&& table_aligns_inf() //
&& table_aligns_neg_inf() //
;
}