diff --git a/LICENSE b/LICENSE index 45b1ea773..71f65b598 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2018-2019 The simdjson authors + Copyright 2018-2023 The simdjson authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/doc/basics.md b/doc/basics.md index 2bba4caf0..856ce4130 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -1418,6 +1418,8 @@ if (error) { It is also important to note that when dealing an invalid number inside a string, simdjson will report a `NUMBER_ERROR` error if the string begins with a number whereas simdjson will report a `INCORRECT_TYPE` error otherwise. +The `*_in_string` methods can also be called on a single document instance: +e.g., when your document consist solely of a quoted number. Dynamic Number Types ------------------------------ diff --git a/doc/implementation-selection.md b/doc/implementation-selection.md index d3ca9ab27..eac7fff27 100644 --- a/doc/implementation-selection.md +++ b/doc/implementation-selection.md @@ -51,7 +51,7 @@ Inspecting the Detected Implementation You can check what implementation is running with `active_implementation`: ```c++ -cout << "simdjson v" << SIMDJSON_STRINGIFY(SIMDJSON_VERSION) << endl; +cout << "simdjson v" << SIMDJSON_VERSION << endl; cout << "Detected the best implementation for your machine: " << simdjson::get_active_implementation()->name(); cout << "(" << simdjson::get_active_implementation()->description() << ")" << endl; ``` diff --git a/include/simdjson/generic/ondemand/document-inl.h b/include/simdjson/generic/ondemand/document-inl.h index 596bad0f4..7459ca363 100644 --- a/include/simdjson/generic/ondemand/document-inl.h +++ b/include/simdjson/generic/ondemand/document-inl.h @@ -67,35 +67,44 @@ simdjson_inline simdjson_result document::get_object() & noexcept { auto value = get_root_value_iterator(); return object::start_root(value); } + +/** + * We decided that calling 'get_double()' on the JSON document '1.233 blabla' should + * give an error, so we check for trailing content. We want to disallow trailing + * content. + * Thus, in several implementations below, we pass a 'true' parameter value to + * a get_root_value_iterator() method: this indicates that we disallow trailing content. + */ + simdjson_inline simdjson_result document::get_uint64() noexcept { - return get_root_value_iterator().get_root_uint64(); + return get_root_value_iterator().get_root_uint64(true); } simdjson_inline simdjson_result document::get_uint64_in_string() noexcept { - return get_root_value_iterator().get_root_uint64_in_string(); + return get_root_value_iterator().get_root_uint64_in_string(true); } simdjson_inline simdjson_result document::get_int64() noexcept { - return get_root_value_iterator().get_root_int64(); + return get_root_value_iterator().get_root_int64(true); } simdjson_inline simdjson_result document::get_int64_in_string() noexcept { - return get_root_value_iterator().get_root_int64_in_string(); + return get_root_value_iterator().get_root_int64_in_string(true); } simdjson_inline simdjson_result document::get_double() noexcept { - return get_root_value_iterator().get_root_double(); + return get_root_value_iterator().get_root_double(true); } simdjson_inline simdjson_result document::get_double_in_string() noexcept { - return get_root_value_iterator().get_root_double_in_string(); + return get_root_value_iterator().get_root_double_in_string(true); } simdjson_inline simdjson_result document::get_string() noexcept { - return get_root_value_iterator().get_root_string(); + return get_root_value_iterator().get_root_string(true); } simdjson_inline simdjson_result document::get_raw_json_string() noexcept { - return get_root_value_iterator().get_root_raw_json_string(); + return get_root_value_iterator().get_root_raw_json_string(true); } simdjson_inline simdjson_result document::get_bool() noexcept { - return get_root_value_iterator().get_root_bool(); + return get_root_value_iterator().get_root_bool(true); } simdjson_inline simdjson_result document::is_null() noexcept { - return get_root_value_iterator().is_root_null(); + return get_root_value_iterator().is_root_null(true); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_array(); } @@ -213,15 +222,15 @@ simdjson_inline bool document::is_negative() noexcept { } simdjson_inline simdjson_result document::is_integer() noexcept { - return get_root_value_iterator().is_root_integer(); + return get_root_value_iterator().is_root_integer(true); } simdjson_inline simdjson_result document::get_number_type() noexcept { - return get_root_value_iterator().get_root_number_type(); + return get_root_value_iterator().get_root_number_type(true); } simdjson_inline simdjson_result document::get_number() noexcept { - return get_root_value_iterator().get_root_number(); + return get_root_value_iterator().get_root_number(true); } @@ -330,14 +339,26 @@ simdjson_inline simdjson_result simdjson_result simdjson_result::get_uint64_in_string() noexcept { + if (error()) { return error(); } + return first.get_uint64_in_string(); +} simdjson_inline simdjson_result simdjson_result::get_int64() noexcept { if (error()) { return error(); } return first.get_int64(); } +simdjson_inline simdjson_result simdjson_result::get_int64_in_string() noexcept { + if (error()) { return error(); } + return first.get_int64_in_string(); +} simdjson_inline simdjson_result simdjson_result::get_double() noexcept { if (error()) { return error(); } return first.get_double(); } +simdjson_inline simdjson_result simdjson_result::get_double_in_string() noexcept { + if (error()) { return error(); } + return first.get_double_in_string(); +} simdjson_inline simdjson_result simdjson_result::get_string() noexcept { if (error()) { return error(); } return first.get_string(); @@ -497,24 +518,41 @@ simdjson_inline document_reference::document_reference(document &d) noexcept : d simdjson_inline void document_reference::rewind() noexcept { doc->rewind(); } simdjson_inline simdjson_result document_reference::get_array() & noexcept { return doc->get_array(); } simdjson_inline simdjson_result document_reference::get_object() & noexcept { return doc->get_object(); } -simdjson_inline simdjson_result document_reference::get_uint64() noexcept { return doc->get_uint64(); } -simdjson_inline simdjson_result document_reference::get_int64() noexcept { return doc->get_int64(); } -simdjson_inline simdjson_result document_reference::get_double() noexcept { return doc->get_double(); } -simdjson_inline simdjson_result document_reference::get_string() noexcept { return doc->get_string(); } -simdjson_inline simdjson_result document_reference::get_raw_json_string() noexcept { return doc->get_raw_json_string(); } -simdjson_inline simdjson_result document_reference::get_bool() noexcept { return doc->get_bool(); } +/** + * The document_reference instances are used primarily/solely for streams of JSON + * documents. + * We decided that calling 'get_double()' on the JSON document '1.233 blabla' should + * give an error, so we check for trailing content. + * + * However, for streams of JSON documents, we want to be able to start from + * "321" "321" "321" + * and parse it successfully as a stream of JSON documents, calling get_uint64_in_string() + * successfully each time. + * + * To achieve this result, we pass a 'false' to a get_root_value_iterator() method: + * this indicates that we allow trailing content. + */ +simdjson_inline simdjson_result document_reference::get_uint64() noexcept { return doc->get_root_value_iterator().get_root_uint64(false); } +simdjson_inline simdjson_result document_reference::get_uint64_in_string() noexcept { return doc->get_root_value_iterator().get_root_uint64_in_string(false); } +simdjson_inline simdjson_result document_reference::get_int64() noexcept { return doc->get_root_value_iterator().get_root_int64(false); } +simdjson_inline simdjson_result document_reference::get_int64_in_string() noexcept { return doc->get_root_value_iterator().get_root_int64_in_string(false); } +simdjson_inline simdjson_result document_reference::get_double() noexcept { return doc->get_root_value_iterator().get_root_double(false); } +simdjson_inline simdjson_result document_reference::get_double_in_string() noexcept { return doc->get_root_value_iterator().get_root_double(false); } +simdjson_inline simdjson_result document_reference::get_string() noexcept { return doc->get_root_value_iterator().get_root_string(false); } +simdjson_inline simdjson_result document_reference::get_raw_json_string() noexcept { return doc->get_root_value_iterator().get_root_raw_json_string(false); } +simdjson_inline simdjson_result document_reference::get_bool() noexcept { return doc->get_root_value_iterator().get_root_bool(false); } simdjson_inline simdjson_result document_reference::get_value() noexcept { return doc->get_value(); } -simdjson_inline simdjson_result document_reference::is_null() noexcept { return doc->is_null(); } +simdjson_inline simdjson_result document_reference::is_null() noexcept { return doc->get_root_value_iterator().is_root_null(false); } #if SIMDJSON_EXCEPTIONS simdjson_inline document_reference::operator array() & noexcept(false) { return array(*doc); } simdjson_inline document_reference::operator object() & noexcept(false) { return object(*doc); } -simdjson_inline document_reference::operator uint64_t() noexcept(false) { return uint64_t(*doc); } -simdjson_inline document_reference::operator int64_t() noexcept(false) { return int64_t(*doc); } -simdjson_inline document_reference::operator double() noexcept(false) { return double(*doc); } +simdjson_inline document_reference::operator uint64_t() noexcept(false) { return get_uint64(); } +simdjson_inline document_reference::operator int64_t() noexcept(false) { return get_int64(); } +simdjson_inline document_reference::operator double() noexcept(false) { return get_double(); } simdjson_inline document_reference::operator std::string_view() noexcept(false) { return std::string_view(*doc); } simdjson_inline document_reference::operator raw_json_string() noexcept(false) { return raw_json_string(*doc); } -simdjson_inline document_reference::operator bool() noexcept(false) { return bool(*doc); } +simdjson_inline document_reference::operator bool() noexcept(false) { return get_bool(); } simdjson_inline document_reference::operator value() noexcept(false) { return value(*doc); } #endif simdjson_inline simdjson_result document_reference::count_elements() & noexcept { return doc->count_elements(); } @@ -533,9 +571,9 @@ simdjson_inline simdjson_result document_reference::is_scalar() noexcept { simdjson_inline simdjson_result document_reference::current_location() noexcept { return doc->current_location(); } simdjson_inline int32_t document_reference::current_depth() const noexcept { return doc->current_depth(); } simdjson_inline bool document_reference::is_negative() noexcept { return doc->is_negative(); } -simdjson_inline simdjson_result document_reference::is_integer() noexcept { return doc->is_integer(); } -simdjson_inline simdjson_result document_reference::get_number_type() noexcept { return doc->get_number_type(); } -simdjson_inline simdjson_result document_reference::get_number() noexcept { return doc->get_number(); } +simdjson_inline simdjson_result document_reference::is_integer() noexcept { return doc->get_root_value_iterator().is_root_integer(false); } +simdjson_inline simdjson_result document_reference::get_number_type() noexcept { return doc->get_root_value_iterator().get_root_number_type(false); } +simdjson_inline simdjson_result document_reference::get_number() noexcept { return doc->get_root_value_iterator().get_root_number(false); } simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json();} @@ -612,14 +650,26 @@ simdjson_inline simdjson_result simdjson_result simdjson_result::get_uint64_in_string() noexcept { + if (error()) { return error(); } + return first.get_uint64_in_string(); +} simdjson_inline simdjson_result simdjson_result::get_int64() noexcept { if (error()) { return error(); } return first.get_int64(); } +simdjson_inline simdjson_result simdjson_result::get_int64_in_string() noexcept { + if (error()) { return error(); } + return first.get_int64_in_string(); +} simdjson_inline simdjson_result simdjson_result::get_double() noexcept { if (error()) { return error(); } return first.get_double(); } +simdjson_inline simdjson_result simdjson_result::get_double_in_string() noexcept { + if (error()) { return error(); } + return first.get_double_in_string(); +} simdjson_inline simdjson_result simdjson_result::get_string() noexcept { if (error()) { return error(); } return first.get_string(); diff --git a/include/simdjson/generic/ondemand/document.h b/include/simdjson/generic/ondemand/document.h index eb401dbaa..9a770c4c7 100644 --- a/include/simdjson/generic/ondemand/document.h +++ b/include/simdjson/generic/ondemand/document.h @@ -577,6 +577,7 @@ protected: friend class field; friend class token; friend class document_stream; + friend class document_reference; }; @@ -593,8 +594,11 @@ public: simdjson_inline simdjson_result get_array() & noexcept; simdjson_inline simdjson_result get_object() & noexcept; simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_uint64_in_string() noexcept; simdjson_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_int64_in_string() noexcept; simdjson_inline simdjson_result get_double() noexcept; + simdjson_inline simdjson_result get_double_in_string() noexcept; simdjson_inline simdjson_result get_string() noexcept; simdjson_inline simdjson_result get_raw_json_string() noexcept; simdjson_inline simdjson_result get_bool() noexcept; @@ -658,9 +662,11 @@ public: simdjson_inline simdjson_result get_array() & noexcept; simdjson_inline simdjson_result get_object() & noexcept; simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_uint64_in_string() noexcept; simdjson_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_int64_in_string() noexcept; simdjson_inline simdjson_result get_double() noexcept; - simdjson_inline simdjson_result get_double_from_string() noexcept; + simdjson_inline simdjson_result get_double_in_string() noexcept; simdjson_inline simdjson_result get_string() noexcept; simdjson_inline simdjson_result get_raw_json_string() noexcept; simdjson_inline simdjson_result get_bool() noexcept; @@ -726,8 +732,11 @@ public: simdjson_inline simdjson_result get_array() & noexcept; simdjson_inline simdjson_result get_object() & noexcept; simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_uint64_in_string() noexcept; simdjson_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_int64_in_string() noexcept; simdjson_inline simdjson_result get_double() noexcept; + simdjson_inline simdjson_result get_double_in_string() noexcept; simdjson_inline simdjson_result get_string() noexcept; simdjson_inline simdjson_result get_raw_json_string() noexcept; simdjson_inline simdjson_result get_bool() noexcept; diff --git a/include/simdjson/generic/ondemand/value_iterator-inl.h b/include/simdjson/generic/ondemand/value_iterator-inl.h index 98c4529dd..9de9b4543 100644 --- a/include/simdjson/generic/ondemand/value_iterator-inl.h +++ b/include/simdjson/generic/ondemand/value_iterator-inl.h @@ -544,7 +544,7 @@ simdjson_inline simdjson_result value_iterator::get_number() noexcept { return num; } -simdjson_inline simdjson_result value_iterator::is_root_integer() noexcept { +simdjson_inline simdjson_result value_iterator::is_root_integer(bool check_trailing) noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("is_root_integer"); uint8_t tmpbuf[20+1]; // <20 digits> is the longest possible unsigned integer @@ -555,12 +555,11 @@ simdjson_inline simdjson_result value_iterator::is_root_integer() noexcept // If the parsing was a success, we must still check that it is // a single scalar. Note that we parse first because of cases like '[]' where // getting TRAILING_CONTENT is wrong. - if((answer.error() == SUCCESS) && (!_json_iter->is_single_token())) { return TRAILING_CONTENT; } + if(check_trailing && (answer.error() == SUCCESS) && (!_json_iter->is_single_token())) { return TRAILING_CONTENT; } return answer; } -simdjson_inline simdjson_result value_iterator::get_root_number_type() noexcept { - if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } +simdjson_inline simdjson_result value_iterator::get_root_number_type(bool check_trailing) noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("number"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, @@ -571,14 +570,11 @@ simdjson_inline simdjson_result logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); return NUMBER_ERROR; } - // If the parsing was a success, we must still check that it is - // a single scalar. Note that we parse first because of cases like '[]' where - // getting TRAILING_CONTENT is wrong. auto answer = numberparsing::get_number_type(tmpbuf); - if((answer.error() == SUCCESS) && (!_json_iter->is_single_token())) { return TRAILING_CONTENT; } + if (check_trailing && (answer.error() == SUCCESS) && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } return answer; } -simdjson_inline simdjson_result value_iterator::get_root_number() noexcept { +simdjson_inline simdjson_result value_iterator::get_root_number(bool check_trailing) noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("number"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, @@ -592,18 +588,21 @@ simdjson_inline simdjson_result value_iterator::get_root_number() noexce number num; error_code error = numberparsing::parse_number(tmpbuf, num); if(error) { return error; } - if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } advance_root_scalar("number"); return num; } - -simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_string() noexcept { - return get_string(); +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_string(bool check_trailing) noexcept { + return get_root_raw_json_string(check_trailing).unescape(json_iter()); } -simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_raw_json_string() noexcept { - return get_raw_json_string(); +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_raw_json_string(bool check_trailing) noexcept { + auto json = peek_scalar("string"); + if (*json != '"') { return incorrect_type_error("Not a string"); } + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_scalar("string"); + return raw_json_string(json+1); } -simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64(bool check_trailing) noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("uint64"); uint8_t tmpbuf[20+1]; // <20 digits> is the longest possible unsigned integer @@ -613,12 +612,12 @@ simdjson_warn_unused simdjson_inline simdjson_result value_iterator::g } auto result = numberparsing::parse_unsigned(tmpbuf); if(result.error() == SUCCESS) { - if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } advance_root_scalar("uint64"); } return result; } -simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64_in_string() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64_in_string(bool check_trailing) noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("uint64"); uint8_t tmpbuf[20+1]; // <20 digits> is the longest possible unsigned integer @@ -628,12 +627,12 @@ simdjson_warn_unused simdjson_inline simdjson_result value_iterator::g } auto result = numberparsing::parse_unsigned_in_string(tmpbuf); if(result.error() == SUCCESS) { - if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } advance_root_scalar("uint64"); } return result; } -simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64(bool check_trailing) noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("int64"); uint8_t tmpbuf[20+1]; // -<19 digits> is the longest possible integer @@ -644,12 +643,12 @@ simdjson_warn_unused simdjson_inline simdjson_result value_iterator::ge auto result = numberparsing::parse_integer(tmpbuf); if(result.error() == SUCCESS) { - if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } advance_root_scalar("int64"); } return result; } -simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64_in_string() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64_in_string(bool check_trailing) noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("int64"); uint8_t tmpbuf[20+1]; // -<19 digits> is the longest possible integer @@ -660,12 +659,12 @@ simdjson_warn_unused simdjson_inline simdjson_result value_iterator::ge auto result = numberparsing::parse_integer_in_string(tmpbuf); if(result.error() == SUCCESS) { - if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } advance_root_scalar("int64"); } return result; } -simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double(bool check_trailing) noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("double"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, @@ -678,13 +677,13 @@ simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get } auto result = numberparsing::parse_double(tmpbuf); if(result.error() == SUCCESS) { - if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } advance_root_scalar("double"); } return result; } -simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double_in_string() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double_in_string(bool check_trailing) noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("double"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, @@ -697,31 +696,32 @@ simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get } auto result = numberparsing::parse_double_in_string(tmpbuf); if(result.error() == SUCCESS) { - if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } advance_root_scalar("double"); } return result; } -simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_bool() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_bool(bool check_trailing) noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("bool"); uint8_t tmpbuf[5+1]; if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) { return incorrect_type_error("Not a boolean"); } auto result = parse_bool(tmpbuf); if(result.error() == SUCCESS) { - if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } advance_root_scalar("bool"); } return result; } -simdjson_inline bool value_iterator::is_root_null() noexcept { - // If there is trailing content, then the document is not null. - if (!_json_iter->is_single_token()) { return false; } +simdjson_inline simdjson_result value_iterator::is_root_null(bool check_trailing) noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("null"); bool result = (max_len >= 4 && !atomparsing::str4ncmp(json, "null") && - (max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[5]))); - if(result) { advance_root_scalar("null"); } + (max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[4]))); + if(result) { // we have something that looks like a null. + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("null"); + } return result; } diff --git a/include/simdjson/generic/ondemand/value_iterator.h b/include/simdjson/generic/ondemand/value_iterator.h index 7cd7a25ea..fe649675f 100644 --- a/include/simdjson/generic/ondemand/value_iterator.h +++ b/include/simdjson/generic/ondemand/value_iterator.h @@ -295,20 +295,20 @@ public: simdjson_warn_unused simdjson_inline simdjson_result get_number_type() noexcept; simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; - simdjson_warn_unused simdjson_inline simdjson_result get_root_string() noexcept; - simdjson_warn_unused simdjson_inline simdjson_result get_root_raw_json_string() noexcept; - simdjson_warn_unused simdjson_inline simdjson_result get_root_uint64() noexcept; - simdjson_warn_unused simdjson_inline simdjson_result get_root_uint64_in_string() noexcept; - simdjson_warn_unused simdjson_inline simdjson_result get_root_int64() noexcept; - simdjson_warn_unused simdjson_inline simdjson_result get_root_int64_in_string() noexcept; - simdjson_warn_unused simdjson_inline simdjson_result get_root_double() noexcept; - simdjson_warn_unused simdjson_inline simdjson_result get_root_double_in_string() noexcept; - simdjson_warn_unused simdjson_inline simdjson_result get_root_bool() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_raw_json_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_uint64(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_uint64_in_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_int64(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_int64_in_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_double(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_double_in_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_bool(bool check_trailing) noexcept; simdjson_warn_unused simdjson_inline bool is_root_negative() noexcept; - simdjson_warn_unused simdjson_inline simdjson_result is_root_integer() noexcept; - simdjson_warn_unused simdjson_inline simdjson_result get_root_number_type() noexcept; - simdjson_warn_unused simdjson_inline simdjson_result get_root_number() noexcept; - simdjson_inline bool is_root_null() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result is_root_integer(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_number_type(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_number(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result is_root_null(bool check_trailing) noexcept; simdjson_inline error_code error() const noexcept; simdjson_inline uint8_t *&string_buf_loc() noexcept; diff --git a/include/simdjson/simdjson_version.h b/include/simdjson/simdjson_version.h index eb6a821e1..978a92c74 100644 --- a/include/simdjson/simdjson_version.h +++ b/include/simdjson/simdjson_version.h @@ -4,7 +4,14 @@ #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION 3.0.1 +/** + * Early versions of simdjson would not put quotes around the string + * which required users to add the quotes. If one examines the most + * common usage of version macros, they are quoted, as they are meant + * to be treated as strings. Thus going forward, the version macro is + * quoted. We do not expect this change to break code. + **/ +#define SIMDJSON_VERSION "3.0.1" namespace simdjson { enum { diff --git a/tests/checkimplementation.cpp b/tests/checkimplementation.cpp index bbe9c62eb..d186d1f93 100644 --- a/tests/checkimplementation.cpp +++ b/tests/checkimplementation.cpp @@ -3,7 +3,7 @@ #include int main(int argc, const char *argv[]) { - std::cout << "simdjson v" << SIMDJSON_STRINGIFY(SIMDJSON_VERSION) << " is running the " << simdjson::get_active_implementation()->name() << " implementation." << std::endl; + std::cout << "simdjson v" << SIMDJSON_VERSION << " is running the " << simdjson::get_active_implementation()->name() << " implementation." << std::endl; const char *expected_implementation = nullptr; if (argc > 1) { expected_implementation = argv[1]; diff --git a/tests/dom/readme_examples.cpp b/tests/dom/readme_examples.cpp index 4abefd97c..816e9004c 100644 --- a/tests/dom/readme_examples.cpp +++ b/tests/dom/readme_examples.cpp @@ -267,7 +267,7 @@ void basics_ndjson_parse_many() { } } void implementation_selection_1() { - cout << "simdjson v" << SIMDJSON_STRINGIFY(SIMDJSON_VERSION) << endl; + cout << "simdjson v" << SIMDJSON_VERSION << endl; cout << "Detected the best implementation for your machine: " << simdjson::get_active_implementation()->name(); cout << "(" << simdjson::get_active_implementation()->description() << ")" << endl; } diff --git a/tests/ondemand/ondemand_document_stream_tests.cpp b/tests/ondemand/ondemand_document_stream_tests.cpp index 777bfb4e6..e55551fba 100644 --- a/tests/ondemand/ondemand_document_stream_tests.cpp +++ b/tests/ondemand/ondemand_document_stream_tests.cpp @@ -714,8 +714,86 @@ namespace document_stream_tests { return (bool_count == 0) && (total_count == 1); } + bool string_with_trailing() { + TEST_START(); + auto json = R"( "a string" badstuff )"_padded; + ondemand::parser parser; + ondemand::document_stream doc; + ASSERT_SUCCESS(parser.iterate_many(json).get(doc)); + std::string_view view; + ASSERT_SUCCESS((*doc.begin()).get_string().get(view)); + ASSERT_EQUAL(view,"a string"); + TEST_SUCCEED(); + } + + bool uint64_with_trailing() { + TEST_START(); + auto json = R"( 133 badstuff )"_padded; + ondemand::parser parser; + ondemand::document_stream doc; + ASSERT_SUCCESS(parser.iterate_many(json).get(doc)); + uint64_t x; + ASSERT_SUCCESS((*doc.begin()).get_uint64().get(x)); + ASSERT_EQUAL(x,133); + TEST_SUCCEED(); + } + + bool int64_with_trailing() { + TEST_START(); + auto json = R"( 133 badstuff )"_padded; + ondemand::parser parser; + ondemand::document_stream doc; + ASSERT_SUCCESS(parser.iterate_many(json).get(doc)); + int64_t x; + ASSERT_SUCCESS((*doc.begin()).get_int64().get(x)); + ASSERT_EQUAL(x,133); + TEST_SUCCEED(); + } + + bool double_with_trailing() { + TEST_START(); + auto json = R"( 133 badstuff )"_padded; + ondemand::parser parser; + ondemand::document_stream doc; + ASSERT_SUCCESS(parser.iterate_many(json).get(doc)); + double x; + ASSERT_SUCCESS((*doc.begin()).get_double().get(x)); + ASSERT_EQUAL(x,133); + TEST_SUCCEED(); + } + + + bool bool_with_trailing() { + TEST_START(); + auto json = R"( true badstuff )"_padded; + ondemand::parser parser; + ondemand::document_stream doc; + ASSERT_SUCCESS(parser.iterate_many(json).get(doc)); + bool x; + ASSERT_SUCCESS((*doc.begin()).get_bool().get(x)); + ASSERT_EQUAL(x,true); + TEST_SUCCEED(); + } + + bool null_with_trailing() { + TEST_START(); + auto json = R"( null badstuff )"_padded; + ondemand::parser parser; + ondemand::document_stream doc; + ASSERT_SUCCESS(parser.iterate_many(json).get(doc)); + bool n; + ASSERT_SUCCESS((*doc.begin()).is_null().get(n)); + ASSERT_EQUAL(n,true); + TEST_SUCCEED(); + } + bool run() { return + string_with_trailing() && + uint64_with_trailing() && + int64_with_trailing() && + bool_with_trailing() && + null_with_trailing() && truncated_utf8() && issue1729() && fuzzaccess() && diff --git a/tests/ondemand/ondemand_number_in_string_tests.cpp b/tests/ondemand/ondemand_number_in_string_tests.cpp index 620edcb62..164304d6e 100644 --- a/tests/ondemand/ondemand_number_in_string_tests.cpp +++ b/tests/ondemand/ondemand_number_in_string_tests.cpp @@ -36,6 +36,93 @@ namespace number_in_string_tests { } )"_padded; + bool in_document_int64() { + TEST_START(); + auto json = R"( "-11232321" )"_padded; + ondemand::parser parser; + auto doc = parser.iterate(json); + int64_t result; + ASSERT_SUCCESS(doc.get_int64_in_string().get(result)); + ASSERT_EQUAL(result,-11232321); + TEST_SUCCEED(); + } + + bool in_document_uint64() { + TEST_START(); + auto json = R"( "11232321" )"_padded; + ondemand::parser parser; + auto doc = parser.iterate(json); + uint64_t result; + ASSERT_SUCCESS(doc.get_uint64_in_string().get(result)); + ASSERT_EQUAL(result,11232321); + TEST_SUCCEED(); + } + + bool in_document_double() { + TEST_START(); + auto json = R"( "11232321.12" )"_padded; + ondemand::parser parser; + auto doc = parser.iterate(json); + double result; + ASSERT_SUCCESS(doc.get_double_in_string().get(result)); + ASSERT_EQUAL(result,11232321.12); + TEST_SUCCEED(); + } + + bool stream_of_pure_int64() { + TEST_START(); + auto json = R"( 1 2 3 )"_padded; + + ondemand::parser parser; + ondemand::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(json).get(stream)); + int document_index = 0; + for (auto doc : stream) { + document_index++; + int64_t result; + ASSERT_SUCCESS(doc.get_int64().get(result)); + ASSERT_EQUAL(result,document_index); + } + ASSERT_EQUAL(3,document_index); + TEST_SUCCEED(); + } + + bool stream_of_direct_int64() { + TEST_START(); + auto json = R"( "1" "2" "3" )"_padded; + + ondemand::parser parser; + ondemand::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(json).get(stream)); + int document_index = 0; + for (auto doc : stream) { + document_index++; + int64_t result; + ASSERT_SUCCESS(doc.get_int64_in_string().get(result)); + ASSERT_EQUAL(result,document_index); + } + ASSERT_EQUAL(3,document_index); + TEST_SUCCEED(); + } + + bool stream_of_int64() { + TEST_START(); + auto json = R"( ["1"] ["2"] ["3"] )"_padded; + + ondemand::parser parser; + ondemand::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(json).get(stream)); + int document_index = 0; + for (auto doc : stream) { + document_index++; + int64_t result; + ASSERT_SUCCESS(doc.get_array().at(0).get_int64_in_string().get(result)); + ASSERT_EQUAL(result,document_index); + } + ASSERT_EQUAL(3,document_index); + TEST_SUCCEED(); + } + bool array_double() { TEST_START(); auto json = R"(["1.2","2.3","-42.3","2.43442e3", "-1.234e3"])"_padded; @@ -255,7 +342,13 @@ namespace number_in_string_tests { } bool run() { - return array_double() && + return stream_of_pure_int64() && + stream_of_direct_int64() && + stream_of_int64() && + in_document_int64() && + in_document_uint64() && + in_document_double() && + array_double() && array_int() && array_unsigned() && object() && diff --git a/tests/ondemand/ondemand_scalar_tests.cpp b/tests/ondemand/ondemand_scalar_tests.cpp index 082369ee9..63869c548 100644 --- a/tests/ondemand/ondemand_scalar_tests.cpp +++ b/tests/ondemand/ondemand_scalar_tests.cpp @@ -297,8 +297,92 @@ namespace scalar_tests { #endif // SIMDJSON_EXCEPTIONS + bool string_with_trailing() { + TEST_START(); + auto json = R"( "a string" badstuff )"_padded; + ondemand::parser parser; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + std::string_view view; + ASSERT_ERROR(doc.get_string().get(view), TRAILING_CONTENT); + TEST_SUCCEED(); + } + + bool uint64_with_trailing() { + TEST_START(); + auto json = R"( 133 badstuff )"_padded; + ondemand::parser parser; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + uint64_t x; + ASSERT_ERROR(doc.get_uint64().get(x), TRAILING_CONTENT); + TEST_SUCCEED(); + } + + bool int64_with_trailing() { + TEST_START(); + auto json = R"( 133 badstuff )"_padded; + ondemand::parser parser; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + int64_t x; + ASSERT_ERROR(doc.get_int64().get(x), TRAILING_CONTENT); + TEST_SUCCEED(); + } + + bool double_with_trailing() { + TEST_START(); + auto json = R"( 133 badstuff )"_padded; + ondemand::parser parser; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + double x; + ASSERT_ERROR(doc.get_double().get(x), TRAILING_CONTENT); + TEST_SUCCEED(); + } + + + bool bool_with_trailing() { + TEST_START(); + auto json = R"( true badstuff )"_padded; + ondemand::parser parser; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + bool x; + ASSERT_ERROR(doc.get_bool().get(x), TRAILING_CONTENT); + TEST_SUCCEED(); + } + + bool null_with_trailing() { + TEST_START(); + auto json = R"( null badstuff )"_padded; + ondemand::parser parser; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ASSERT_ERROR(doc.is_null(), TRAILING_CONTENT); + TEST_SUCCEED(); + } + + bool nully() { + TEST_START(); + auto json = R"( nully )"_padded; + ondemand::parser parser; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + bool x; + ASSERT_SUCCESS(doc.is_null().get(x)); + ASSERT_TRUE(!x); + TEST_SUCCEED(); + } + bool run() { return + nully() && + string_with_trailing() && + uint64_with_trailing() && + int64_with_trailing() && + bool_with_trailing() && + null_with_trailing() && string_value() && numeric_values() && boolean_values() && diff --git a/tools/json2json.cpp b/tools/json2json.cpp index 0b776dc0f..ed641c639 100644 --- a/tools/json2json.cpp +++ b/tools/json2json.cpp @@ -20,7 +20,7 @@ int main(int argc, const char *argv[]) { std::string progName = "json2json"; std::string progUsage = "json2json version "; - progUsage += SIMDJSON_STRINGIFY(SIMDJSON_VERSION); + progUsage += SIMDJSON_VERSION; progUsage += " ("; progUsage += simdjson::get_active_implementation()->name(); progUsage += ")\n";