diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8e098fdd5..da4962bab 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -3,6 +3,7 @@ link_libraries(simdjson-internal-flags test-data simdjson-windows-headers) include(${PROJECT_SOURCE_DIR}/cmake/add_cpp_test.cmake) add_subdirectory(dom) add_subdirectory(ondemand) +add_subdirectory(singlestage) # All remaining tests link with simdjson proper diff --git a/tests/singlestage/CMakeLists.txt b/tests/singlestage/CMakeLists.txt new file mode 100644 index 000000000..61fa737d7 --- /dev/null +++ b/tests/singlestage/CMakeLists.txt @@ -0,0 +1,40 @@ +# All remaining tests link with simdjson proper +link_libraries(simdjson) +include_directories(..) +add_subdirectory(compilation_failure_tests) +add_cpp_test(singlestage_log_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_log_error_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_tostring_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_active_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_array_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_array_error_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_compilation_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_document_stream_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_error_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_error_location_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_json_pointer_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_key_string_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_misc_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_number_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_number_in_string_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_object_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_object_error_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_ordering_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_parse_api_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_readme_examples LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_scalar_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_twitter_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_wrong_type_error_tests LABELS singlestage acceptance per_implementation) +add_cpp_test(singlestage_iterate_many_csv LABELS singlestage acceptance per_implementation) + +if(HAVE_POSIX_FORK AND HAVE_POSIX_WAIT) # assert tests use fork and wait, which aren't on MSVC + add_cpp_test(singlestage_assert_out_of_order_values LABELS assert per_implementation explicitonly singlestage) +endif() + +# Copy the simdjson dll into the tests directory +if(MSVC AND BUILD_SHARED_LIBS) + add_custom_command(TARGET singlestage_parse_api_tests POST_BUILD # Adds a post-build event + COMMAND ${CMAKE_COMMAND} -E copy_if_different # which executes "cmake -E copy_if_different..." + "$" # <--this is in-file + "$") # <--this is out-file path +endif(MSVC AND BUILD_SHARED_LIBS) diff --git a/tests/singlestage/compilation_failure_tests/CMakeLists.txt b/tests/singlestage/compilation_failure_tests/CMakeLists.txt new file mode 100644 index 000000000..8f1418fe0 --- /dev/null +++ b/tests/singlestage/compilation_failure_tests/CMakeLists.txt @@ -0,0 +1,20 @@ +# +# This directory contains files aimed to verify that constructs that +# are supposed to fail at compile time, indeed do so. +# To prevent bit rot, the same source file is compiled twice with +# the macro COMPILATION_TEST_USE_FAILING_CODE set to 0 or 1. +# + +# adds a compilation test. Two targets are created, one expected to +# succeed compilation and one that is expected to fail. +function(add_dual_compile_test TEST_NAME) + add_cpp_test(${TEST_NAME}_should_compile SOURCES ${TEST_NAME}.cpp COMPILE_ONLY LABELS singlestage no_mingw) + add_cpp_test(${TEST_NAME}_should_not_compile SOURCES ${TEST_NAME}.cpp COMPILE_ONLY WILL_FAIL LABELS singlestage acceptance no_mingw) + target_compile_definitions(${TEST_NAME}_should_not_compile PRIVATE COMPILATION_TEST_USE_FAILING_CODE=1) +endfunction(add_dual_compile_test) + + +add_dual_compile_test(singlestage_iterate_char_star) +add_dual_compile_test(singlestage_iterate_string_view) +add_dual_compile_test(singlestage_iterate_temporary_buffer) +add_dual_compile_test(singlestage_first_second_access) diff --git a/tests/singlestage/compilation_failure_tests/singlestage_first_second_access.cpp b/tests/singlestage/compilation_failure_tests/singlestage_first_second_access.cpp new file mode 100644 index 000000000..8b7a2288f --- /dev/null +++ b/tests/singlestage/compilation_failure_tests/singlestage_first_second_access.cpp @@ -0,0 +1,28 @@ + +#include +#include "simdjson.h" + +using namespace simdjson; + +int main() { + auto json = "1"_padded; + singlestage::parser parser; + singlestage::document doc; + auto error = parser.iterate(json).get(doc); + if(error) { return EXIT_FAILURE; } + simdjson_result query = doc.at_pointer("/"); +#if COMPILATION_TEST_USE_FAILING_CODE + if(query.second == simdjson::SUCCESS) { + std::cout << "success" << std::endl; + std::cout << query.first << std::endl; + } +#else + singlestage::value val; + error = query.get(val); + if(error == simdjson::SUCCESS) { + std::cout << "success" << std::endl; + std::cout << val << std::endl; + } +#endif + return EXIT_SUCCESS; +} \ No newline at end of file diff --git a/tests/singlestage/compilation_failure_tests/singlestage_iterate_char_star.cpp b/tests/singlestage/compilation_failure_tests/singlestage_iterate_char_star.cpp new file mode 100644 index 000000000..93c4db1f0 --- /dev/null +++ b/tests/singlestage/compilation_failure_tests/singlestage_iterate_char_star.cpp @@ -0,0 +1,19 @@ +#include +#include "simdjson.h" + +using namespace simdjson; + +int main() { + singlestage::parser parser; +#if COMPILATION_TEST_USE_FAILING_CODE + const char* json; + auto doc = parser.iterate(json, strlen(json)); +#else + auto json = "1"_padded; + auto doc = parser.iterate(json); +#endif + int64_t value; + auto error = doc.get(value); + if (error) { exit(1); } + std::cout << value << std::endl; +} diff --git a/tests/singlestage/compilation_failure_tests/singlestage_iterate_string_view.cpp b/tests/singlestage/compilation_failure_tests/singlestage_iterate_string_view.cpp new file mode 100644 index 000000000..441caa7ab --- /dev/null +++ b/tests/singlestage/compilation_failure_tests/singlestage_iterate_string_view.cpp @@ -0,0 +1,19 @@ +#include +#include "simdjson.h" + +using namespace simdjson; + +int main() { + singlestage::parser parser; +#if COMPILATION_TEST_USE_FAILING_CODE + auto json = std::string_view("1"); + auto doc = parser.iterate(json); +#else + auto json = "1"_padded; + auto doc = parser.iterate(json); +#endif + int64_t value; + auto error = doc.get(value); + if (error) { exit(1); } + std::cout << value << std::endl; +} diff --git a/tests/singlestage/compilation_failure_tests/singlestage_iterate_temporary_buffer.cpp b/tests/singlestage/compilation_failure_tests/singlestage_iterate_temporary_buffer.cpp new file mode 100644 index 000000000..2e314167b --- /dev/null +++ b/tests/singlestage/compilation_failure_tests/singlestage_iterate_temporary_buffer.cpp @@ -0,0 +1,18 @@ +#include +#include "simdjson.h" + +using namespace simdjson; + +int main() { + singlestage::parser parser; +#if COMPILATION_TEST_USE_FAILING_CODE + auto doc = parser.iterate("1"_padded); +#else + auto json = "1"_padded; + auto doc = parser.iterate(json); +#endif + int64_t value; + auto error = doc.get(value); + if (error) { exit(1); } + std::cout << value << std::endl; +} diff --git a/tests/singlestage/singlestage_active_tests.cpp b/tests/singlestage/singlestage_active_tests.cpp new file mode 100644 index 000000000..121acd976 --- /dev/null +++ b/tests/singlestage/singlestage_active_tests.cpp @@ -0,0 +1,71 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace active_tests { + +#if SIMDJSON_EXCEPTIONS + + bool parser_child() { + TEST_START(); + singlestage::parser parser; + const padded_string json = R"({ "parent": {"child1": {"name": "John"} , "child2": {"name": "Daniel"}} })"_padded; + auto doc = parser.iterate(json); + singlestage::object parent = doc["parent"]; + { + singlestage::object c1 = parent["child1"]; + if(std::string_view(c1["name"]) != "John") { return false; } + } + { + singlestage::object c2 = parent["child2"]; + if(std::string_view(c2["name"]) != "Daniel") { return false; } + } + return true; + } + + bool parser_doc_correct() { + TEST_START(); + singlestage::parser parser; + const padded_string json = R"({ "key1": 1, "key2":2, "key3": 3 })"_padded; + auto doc = parser.iterate(json); + singlestage::object root_object = doc.get_object(); + int64_t k1 = root_object["key1"]; + int64_t k2 = root_object["key2"]; + int64_t k3 = root_object["key3"]; + return (k1 == 1) && (k2 == 2) && (k3 == 3); + } + + bool parser_doc_limits() { + TEST_START(); + singlestage::parser parser; + const padded_string json = R"({ "key1": 1, "key2":2, "key3": 3 })"_padded; + auto doc = parser.iterate(json); + int64_t k1 = doc["key1"]; + try { + int64_t k2 = doc["key2"]; + (void) k2; + } catch (simdjson::simdjson_error &) { + return true; // we expect to fail. + } + (void) k1; + return false; + } + +#endif // SIMDJSON_EXCEPTIONS + + bool run() { + return +#if SIMDJSON_EXCEPTIONS + parser_child() && + parser_doc_correct() && + // parser_doc_limits() && // Failure is dependent on build type here ... +#endif // SIMDJSON_EXCEPTIONS + true; + } + +} // namespace active_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, active_tests::run); +} diff --git a/tests/singlestage/singlestage_array_error_tests.cpp b/tests/singlestage/singlestage_array_error_tests.cpp new file mode 100644 index 000000000..f8427f364 --- /dev/null +++ b/tests/singlestage/singlestage_array_error_tests.cpp @@ -0,0 +1,216 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace array_error_tests { + using namespace std; + + template + bool assert_iterate(T array, V *expected, size_t N, simdjson::error_code *expected_error, size_t N2) { + /** + * We use printouts because the assert_iterate is abstract and hard to + * understand intuitively. + */ + std::cout << " --- assert_iterate "; + size_t count = 0; + for (auto elem : std::forward(array)) { + std::cout << "-"; std::cout.flush(); + V actual{}; + auto actual_error = elem.get(actual); + if (count >= N) { + if (count >= (N+N2)) { + std::cerr << "FAIL: Extra error reported: " << actual_error << std::endl; + return false; + } + std::cout << "[ expect: " << expected_error[count - N] << " ]"; std::cout.flush(); + ASSERT_ERROR(actual_error, expected_error[count - N]); + } else { + std::cout << "[ expect: SUCCESS ]"; std::cout.flush(); + ASSERT_SUCCESS(actual_error); + std::cout << "{ expect value : "<< expected[count] << " }"; std::cout.flush(); + + ASSERT_EQUAL(actual, expected[count]); + } + count++; + } + ASSERT_EQUAL(count, N+N2); + std::cout << std::endl; + return true; + } + + template + bool assert_iterate(T &array, V (&&expected)[N], simdjson::error_code (&&expected_error)[N2]) { + return assert_iterate(array, expected, N, expected_error, N2); + } + + template + bool assert_iterate(T &array, simdjson::error_code (&&expected_error)[N2]) { + return assert_iterate(array, nullptr, 0, expected_error, N2); + } + + template + bool assert_iterate(T &array, V (&&expected)[N]) { + return assert_iterate(array, expected, N, nullptr, 0); + } + + template + bool assert_iterate(T &&array, V (&&expected)[N], simdjson::error_code (&&expected_error)[N2]) { + return assert_iterate(std::forward(array), expected, N, expected_error, N2); + } + + template + bool assert_iterate(T &&array, simdjson::error_code (&&expected_error)[N2]) { + return assert_iterate(std::forward(array), nullptr, 0, expected_error, N2); + } + + template + bool assert_iterate(T &&array, V (&&expected)[N]) { + return assert_iterate(std::forward(array), expected, N, nullptr, 0); + } + + bool top_level_array_iterate_error() { + TEST_START(); + SINGLESTAGE_SUBTEST("missing comma", "[1 1]", assert_iterate(doc, { int64_t(1) }, { TAPE_ERROR })); + SINGLESTAGE_SUBTEST("extra comma ", "[1,,1]", assert_iterate(doc, { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR })); + SINGLESTAGE_SUBTEST("extra comma ", "[,]", assert_iterate(doc, { INCORRECT_TYPE, TAPE_ERROR })); + SINGLESTAGE_SUBTEST("extra comma ", "[,,]", assert_iterate(doc, { INCORRECT_TYPE, TAPE_ERROR })); + TEST_SUCCEED(); + } + bool top_level_array_iterate_unclosed_error() { + TEST_START(); + SINGLESTAGE_SUBTEST("unclosed extra comma", "[,", assert_iterate(doc, { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed ", "[1 ", assert_iterate(doc, { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed extra comma", "[,,", assert_iterate(doc, { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed ", "[1,", assert_iterate(doc, { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed ", "[1", assert_iterate(doc, { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed ", "[", assert_iterate(doc, { INCOMPLETE_ARRAY_OR_OBJECT })); + TEST_SUCCEED(); + } + + bool array_iterate_error() { + TEST_START(); + SINGLESTAGE_SUBTEST("missing comma", R"({ "a": [1 1] })", assert_iterate(doc["a"], { int64_t(1) }, { TAPE_ERROR })); + SINGLESTAGE_SUBTEST("extra comma ", R"({ "a": [1,,1] })", assert_iterate(doc["a"], { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR })); + SINGLESTAGE_SUBTEST("extra comma ", R"({ "a": [1,,] })", assert_iterate(doc["a"], { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR })); + SINGLESTAGE_SUBTEST("extra comma ", R"({ "a": [,] })", assert_iterate(doc["a"], { INCORRECT_TYPE, TAPE_ERROR})); + SINGLESTAGE_SUBTEST("extra comma ", R"({ "a": [,,] })", assert_iterate(doc["a"], { INCORRECT_TYPE, TAPE_ERROR })); + TEST_SUCCEED(); + } + bool array_iterate_unclosed_error() { + TEST_START(); + SINGLESTAGE_SUBTEST("unclosed extra comma", R"({ "a": [,)", assert_iterate(doc["a"], { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed extra comma", R"({ "a": [,,)", assert_iterate(doc["a"], { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed ", R"({ "a": [1 )", assert_iterate(doc["a"], { INCOMPLETE_ARRAY_OR_OBJECT })); + // TODO These pass the user values that may run past the end of the buffer if they aren't careful + // In particular, if the padding is decorated with the wrong values, we could cause overrun! + SINGLESTAGE_SUBTEST("unclosed ", R"({ "a": [1,)", assert_iterate(doc["a"], { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed ", R"({ "a": [1)", assert_iterate(doc["a"], { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed ", R"({ "a": [)", assert_iterate(doc["a"], { INCOMPLETE_ARRAY_OR_OBJECT })); + TEST_SUCCEED(); + } + bool array_iterate_incomplete_error() { + TEST_START(); +#if SIMDJSON_CHECK_EOF + SINGLESTAGE_SUBTEST("unclosed after array", R"([ [1] )", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed after array", R"([ [1,])", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR })); + SINGLESTAGE_SUBTEST("unclosed after array", R"([ [1])", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed after array", R"([ [])", assert_iterate(doc.get_array().at(0), { INCOMPLETE_ARRAY_OR_OBJECT })); +#else + SINGLESTAGE_SUBTEST("unclosed after array", R"([ [1] )", assert_iterate(doc.get_array().at(0), { int64_t(1) })); + SINGLESTAGE_SUBTEST("unclosed after array", R"([ [1,])", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR })); + SINGLESTAGE_SUBTEST("unclosed after array", R"([ [1])", assert_iterate(doc.get_array().at(0), { int64_t(1) })); +#endif + TEST_SUCCEED(); + } + +#ifdef SIMDJSON_DEVELOPMENT_CHECKS + bool out_of_order_array_iteration_error() { + TEST_START(); + auto json = R"([ [ 1, 2 ] ])"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + for (auto arr : doc) { + for (auto subelement : arr) { ASSERT_SUCCESS(subelement); } + ASSERT_ITERATE_ERROR( arr, OUT_OF_ORDER_ITERATION ); + } + return true; + })); + SUBTEST("value", test_singlestage_doc(json, [&](auto doc) { + for (auto element : doc) { + singlestage::value arr; + ASSERT_SUCCESS( element.get(arr) ); + for (auto subelement : arr) { ASSERT_SUCCESS(subelement); } + ASSERT_ITERATE_ERROR( arr, OUT_OF_ORDER_ITERATION ); + } + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + for (auto element : doc) { + auto arr = element.get_array(); + for (auto subelement : arr) { ASSERT_SUCCESS(subelement); } + ASSERT_ITERATE_ERROR( arr, OUT_OF_ORDER_ITERATION ); + } + return true; + })); + SUBTEST("array", test_singlestage_doc(json, [&](auto doc) { + for (auto element : doc) { + singlestage::array arr; + ASSERT_SUCCESS( element.get(arr) ); + for (auto subelement : arr) { ASSERT_SUCCESS(subelement); } + ASSERT_ITERATE_ERROR( arr, OUT_OF_ORDER_ITERATION ); + } + return true; + })); + TEST_SUCCEED(); + } + + bool out_of_order_top_level_array_iteration_error() { + TEST_START(); + auto json = R"([ 1, 2 ])"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto arr) { + for (auto element : arr) { ASSERT_SUCCESS(element); } + ASSERT_ITERATE_ERROR( arr, OUT_OF_ORDER_ITERATION ); + return true; + })); + SUBTEST("document", test_singlestage_doc(json, [&](auto doc) { + singlestage::document arr; + ASSERT_SUCCESS( std::move(doc).get(arr) ); + for (auto element : arr) { ASSERT_SUCCESS(element); } + ASSERT_ITERATE_ERROR( arr, OUT_OF_ORDER_ITERATION ); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + simdjson_result arr = doc.get_array(); + for (auto element : arr) { ASSERT_SUCCESS(element); } + ASSERT_ITERATE_ERROR( arr, OUT_OF_ORDER_ITERATION ); + return true; + })); + SUBTEST("array", test_singlestage_doc(json, [&](auto doc) { + singlestage::array arr; + ASSERT_SUCCESS( doc.get(arr) ); + for (auto element : arr) { ASSERT_SUCCESS(element); } + ASSERT_ITERATE_ERROR( arr, OUT_OF_ORDER_ITERATION ); + return true; + })); + TEST_SUCCEED(); + } +#endif // SIMDJSON_DEVELOPMENT_CHECKS + + bool run() { + return + top_level_array_iterate_error() && + top_level_array_iterate_unclosed_error() && + array_iterate_error() && + array_iterate_unclosed_error() && + array_iterate_incomplete_error() && +#ifdef SIMDJSON_DEVELOPMENT_CHECKS + out_of_order_array_iteration_error() && + out_of_order_top_level_array_iteration_error() && +#endif + true; + } +} + +int main(int argc, char *argv[]) { + return test_main(argc, argv, array_error_tests::run); +} diff --git a/tests/singlestage/singlestage_array_tests.cpp b/tests/singlestage/singlestage_array_tests.cpp new file mode 100644 index 000000000..3bb14e4db --- /dev/null +++ b/tests/singlestage/singlestage_array_tests.cpp @@ -0,0 +1,875 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace array_tests { + using namespace std; + using simdjson::singlestage::json_type; + bool issue1977() { + TEST_START(); + auto json = R"([1, 2] foo ])"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::array array; + ASSERT_SUCCESS(doc.get_array().get(array)); + for (auto values : array) { + ASSERT_SUCCESS(values); + } + ASSERT_FALSE(doc.at_end()); + TEST_SUCCEED(); + } + bool issue1588() { + TEST_START(); + const auto json = R"({ + "nodes" : [ + { + "rotation" : [ + 0.16907575726509094, + 0.7558803558349609, + -0.27217137813568115, + 0.570947527885437 + ], + "translation" : [ + 4.076245307922363, + 5.903861999511719, + -1.0054539442062378 + ] + }, + { + "camera" : 0, + "rotation" : [ + -0.7071067690849304, + 0, + 0, + 0.7071067690849304 + ] + }, + { + "children" : [ + 1 + ], + "translation" : [ + 7.358891487121582, + 4.958309173583984, + 6.925790786743164 + ] + }, + { + "mesh" : 1, + "scale" : [ + 4.7498908042907715, + 4.7498908042907715, + 4.7498908042907715 + ] + } + ] +})"_padded; + // we query 'rotation', 'scale', 'translation' in sequence + const bool expected_value[][3] = { {true, false, true}, + {true, false, false}, {false, false, true}, {false, true, false} }; + + SUBTEST("singlestage::issue1588::sequence", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::array array; + ASSERT_SUCCESS( doc_result["nodes"].get(array) ); + + size_t i=0; + for (auto value : array) { + singlestage::object current_object; + ASSERT_SUCCESS( value.get_object().get(current_object) ); + simdjson::singlestage::array rotation; + if(expected_value[i][0]) { + ASSERT_SUCCESS( current_object["rotation"].get(rotation) ); + } else { + ASSERT_ERROR( current_object["rotation"].get(rotation), NO_SUCH_FIELD ); + } + simdjson::singlestage::array scale; + if(expected_value[i][1]) { + ASSERT_SUCCESS( current_object["scale"].get(scale) ); + } else { + ASSERT_ERROR( current_object["scale"].get(scale), NO_SUCH_FIELD ); + } + simdjson::singlestage::array translation; + if(expected_value[i][2]) { + ASSERT_SUCCESS( current_object["translation"].get(translation) ); + } else { + ASSERT_ERROR( current_object["translation"].get(translation), NO_SUCH_FIELD ); + } + i++; + } + ASSERT_EQUAL(i, 4); + return true; + })); + SUBTEST("singlestage::issue1588::originalcode", test_singlestage_doc(json, [&](auto doc_result) { + int count_nodes = 0; + simdjson::singlestage::array doc_nodes; + auto error = doc_result["nodes"].get(doc_nodes); + ASSERT_SUCCESS( error ); + for (auto node_iterator : doc_nodes) { + singlestage::object node_obj; + ASSERT_SUCCESS( node_iterator.get_object().get(node_obj) ); + simdjson::singlestage::array rotation; + std::cout << "checking rotation" << std::endl; + if(expected_value[count_nodes][0]) { + ASSERT_SUCCESS( node_obj["rotation"].get(rotation) ); + } else { + ASSERT_ERROR( node_obj["rotation"].get(rotation), NO_SUCH_FIELD ); + } + + simdjson::singlestage::array scale; + std::cout << "checking scale" << std::endl; + if(expected_value[count_nodes][1]) { + ASSERT_SUCCESS( node_obj["scale"].get(scale) ); + } else { + ASSERT_ERROR( node_obj["scale"].get(scale), NO_SUCH_FIELD ); + } + + simdjson::singlestage::array translation; + std::cout << "checking translation" << std::endl; + if (!error) { std::cout << "translation!\n"; } + if(expected_value[count_nodes][2]) { + ASSERT_SUCCESS( node_obj["translation"].get(translation) ); + } else { + ASSERT_ERROR( node_obj["translation"].get(translation), NO_SUCH_FIELD ); + } + + ++count_nodes; + } + ASSERT_EQUAL(count_nodes, 4); + return true; + })); + TEST_SUCCEED(); + } + + bool issue1876() { + TEST_START(); + auto json = R"( [] )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + size_t count{}; + ASSERT_SUCCESS(doc.count_elements().get(count)); + singlestage::array arr; + ASSERT_SUCCESS(doc.get_array().get(arr)); + + ASSERT_EQUAL(count, 0); + for (auto element : arr) { + double value; + ASSERT_SUCCESS(element.get_double().get(value)); + std::cout << value << std::endl; + } + TEST_SUCCEED(); + } + + bool iterate_complex_array_count() { + TEST_START(); + singlestage::parser parser; + auto cars_json = R"( { "zero":[], "test":[ { "val1":1, "val2":2 }, { "val1":1, "val2":2 } ] } )"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(cars_json).get(doc)); + singlestage::array firstmyarray; + singlestage::array secondmyarray; + ASSERT_SUCCESS(doc.find_field("zero").get_array().get(firstmyarray)); + size_t count{0}; + ASSERT_SUCCESS(firstmyarray.count_elements().get(count)); + ASSERT_EQUAL(count, 0); + size_t new_count{0}; + for(simdjson_unused auto elem: firstmyarray) { new_count++; } + ASSERT_EQUAL(new_count, 0); + ASSERT_SUCCESS(doc.find_field("test").get_array().get(secondmyarray)); + ASSERT_SUCCESS(secondmyarray.count_elements().get(count)); + ASSERT_EQUAL(count, 2); + new_count = 0; + for(simdjson_unused auto elem: secondmyarray) { new_count++; } + ASSERT_EQUAL(count, new_count); + TEST_SUCCEED(); + } + + bool iterate_sub_array_count() { + TEST_START(); + singlestage::parser parser; + auto key_value_json = R"( { "test":[ 1,2,3], "joe": [1,2] } )"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(key_value_json).get(doc)); + singlestage::object obj; + ASSERT_SUCCESS(doc.get_object().get(obj)); + singlestage::value v; + ASSERT_SUCCESS(doc.find_field("test").get(v)); + size_t count{}; + ASSERT_SUCCESS(v.count_elements().get(count)); + ASSERT_EQUAL(count, 3); + ASSERT_SUCCESS(doc.find_field("joe").get(v)); + ASSERT_SUCCESS(v.count_elements().get(count)); + ASSERT_EQUAL(count, 2); + TEST_SUCCEED(); + } + + bool iterate_array_count() { + TEST_START(); + const auto json = R"([ 1, 10, 100 ])"_padded; + const vector expected_value = { 1, 10, 100 }; + + SUBTEST("singlestage::count_elements", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::array array; + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_SUCCESS( doc_result.get_array().get(array) ); + size_t count{}; + ASSERT_SUCCESS( array.count_elements().get(count) ); + ASSERT_EQUAL(count, expected_value.size()); + return true; + })); + SUBTEST("singlestage::count_elements_and_decode", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::array array; + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_SUCCESS( doc_result.get(array) ); + size_t count{}; + ASSERT_SUCCESS( array.count_elements().get(count) ); + ASSERT_EQUAL(count, expected_value.size()); + size_t i = 0; + std::vector receiver(count); + for (auto value : array) { + uint64_t actual{}; + ASSERT_SUCCESS( value.get(actual) ); + ASSERT_EQUAL(actual, expected_value[i]); + receiver[i] = actual; + i++; + } + return true; + })); + TEST_SUCCEED(); + } + + + bool iterate_empty_array_count() { + TEST_START(); + const auto json = R"([])"_padded; + + SUBTEST("singlestage::count_elements", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::array array; + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_SUCCESS( doc_result.get_array().get(array) ); + size_t count{}; + ASSERT_SUCCESS( array.count_elements().get(count) ); + ASSERT_EQUAL(count, 0); + return true; + })); + SUBTEST("singlestage::count_elements_and_decode", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::array array; + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_SUCCESS( doc_result.get(array) ); + size_t count{}; + ASSERT_SUCCESS( array.count_elements().get(count) ); + ASSERT_EQUAL(count, 0); + size_t i = 0; + std::vector receiver(count); + for (auto value : array) { + uint64_t actual{}; + ASSERT_SUCCESS( value.get(actual) ); + i++; + } + ASSERT_EQUAL(i, 0); + return true; + })); + TEST_SUCCEED(); + } + + bool iterate_bad_array_count() { + TEST_START(); + const auto badjson = R"([ 1, 10 100 ])"_padded; + + + SUBTEST("singlestage::count_elements", test_singlestage_doc(badjson, [&](auto doc_result) { + singlestage::array array; + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_SUCCESS( doc_result.get(array) ); + size_t count{}; + auto e = array.count_elements().get(count); + if( e != TAPE_ERROR) { + std::cout << e << "\n"; + std::cout << "expected: " << TAPE_ERROR << "\n"; + std::cout << "count = " << count << "\n"; + return false; + } + return true; + })); + TEST_SUCCEED(); + } + bool iterate_document_array_count() { + TEST_START(); + auto empty = R"( [] )"_padded; + SUBTEST("singlestage::empty_doc_array", test_singlestage_doc(empty, [&](auto doc_result) { + size_t count{}; + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_SUCCESS( doc_result.count_elements().get(count) ); + ASSERT_EQUAL( count, 0 ); + return true; + })); + auto basic = R"( [-1.234, 100000000000000, null, [1,2,3], {"t":true, "f":false}] )"_padded; + SUBTEST("singlestage::basic_doc_array", test_singlestage_doc(basic, [&](auto doc_result) { + size_t count{}; + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_SUCCESS( doc_result.count_elements().get(count) ); + ASSERT_EQUAL( count, 5 ); + return true; + })); + TEST_SUCCEED(); + } + bool iterate_bad_document_array_count() { + TEST_START(); + padded_string bad_jsons[2] = {R"( [1, 10 1000] )"_padded, R"( [1.23, 2.34 )"_padded}; + std::string names[2] = {"missing_comma", "missing_bracket"}; + simdjson::error_code errors[2] = {TAPE_ERROR, INCOMPLETE_ARRAY_OR_OBJECT}; + size_t count{0}; + + for (auto name : names) { + SUBTEST("singlestage::" + name, test_singlestage_doc(bad_jsons[count], [&](auto doc_result) { + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_ERROR(doc_result.count_elements(), errors[count]); + return true; + })); + count++; + } + ASSERT_EQUAL(count, 2); + TEST_SUCCEED(); + } + bool iterate_document_array() { + TEST_START(); + const auto json = R"([ 1, 10, 100 ])"_padded; + const uint64_t expected_value[] = { 1, 10, 100 }; + + SUBTEST("singlestage::array", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::array array; + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_SUCCESS( doc_result.get(array) ); + + size_t i = 0; + for (auto value : array) { + int64_t actual{}; + ASSERT_SUCCESS( value.get(actual) ); + ASSERT_EQUAL(actual, expected_value[i]); + i++; + } + ASSERT_EQUAL(i*sizeof(int64_t), sizeof(expected_value)); + return true; + })); + + SUBTEST("singlestage::array-document-rewind", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::array array; + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_SUCCESS( doc_result.get(array) ); + + size_t i = 0; + for (auto value : array) { (void)value; i++; } + ASSERT_EQUAL(i*sizeof(uint64_t), sizeof(expected_value)); + std::vector container(i); // container of size 'i'. + + doc_result.rewind(); + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_SUCCESS( doc_result.get(array) ); + i = 0; + for (auto value : array) { + int64_t actual{}; + ASSERT_SUCCESS( value.get(actual) ); + container[i] = actual; + i++; + } + ASSERT_EQUAL(i * sizeof(int64_t), sizeof(expected_value)); + for(size_t j = 0; j < sizeof(expected_value)/sizeof(int64_t); j++) { + ASSERT_EQUAL(container[j], expected_value[j]); + } + return true; + })); + SUBTEST("singlestage::array-rewind", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::array array; + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_SUCCESS( doc_result.get(array) ); + + size_t i = 0; + for (auto value : array) { (void)value; i++; } + ASSERT_EQUAL(i*sizeof(uint64_t), sizeof(expected_value)); + std::vector container(i); // container of size 'i'. + + array.reset(); + i = 0; + for (auto value : array) { + int64_t actual{}; + ASSERT_SUCCESS( value.get(actual) ); + container[i] = actual; + i++; + } + ASSERT_EQUAL(i * sizeof(int64_t), sizeof(expected_value)); + for(size_t j = 0; j < sizeof(expected_value)/sizeof(int64_t); j++) { + ASSERT_EQUAL(container[j], expected_value[j]); + } + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + simdjson_result array = doc_result.get_array(); + ASSERT_RESULT( doc_result.type(), json_type::array ); + size_t i=0; + for (simdjson_unused auto value : array) { int64_t actual; ASSERT_SUCCESS( value.get(actual) ); ASSERT_EQUAL(actual, expected_value[i]); i++; } + ASSERT_EQUAL(i*sizeof(uint64_t), sizeof(expected_value)); + return true; + })); + + SUBTEST("singlestage::document", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::document doc; + ASSERT_SUCCESS( std::move(doc_result).get(doc) ); + ASSERT_RESULT( doc.type(), json_type::array ); + size_t i=0; + for (simdjson_unused auto value : doc) { int64_t actual; ASSERT_SUCCESS( value.get(actual) ); ASSERT_EQUAL(actual, expected_value[i]); i++; } + ASSERT_EQUAL(i*sizeof(uint64_t), sizeof(expected_value)); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + size_t i=0; + ASSERT_RESULT( doc_result.type(), json_type::array ); + for (simdjson_unused auto value : doc_result) { int64_t actual; ASSERT_SUCCESS( value.get(actual) ); ASSERT_EQUAL(actual, expected_value[i]); i++; } + ASSERT_EQUAL(i*sizeof(uint64_t), sizeof(expected_value)); + return true; + })); + TEST_SUCCEED(); + } + bool empty_rewind() { + TEST_START(); + const auto json = R"( [] )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::array arr; + ASSERT_SUCCESS(doc.get_array().get(arr)); + for(simdjson_unused auto i : arr) { + TEST_FAIL("should be empty?"); + } + arr.reset(); + for(simdjson_unused auto i : arr) { + TEST_FAIL("should be empty?"); + } + TEST_SUCCEED(); + } + bool count_empty(simdjson::singlestage::array arr) { + size_t count{}; + ASSERT_SUCCESS(arr.count_elements().get(count)); + ASSERT_EQUAL(count, 0); + bool is_empty; + ASSERT_SUCCESS(arr.is_empty().get(is_empty)); + ASSERT_TRUE(is_empty); + return true; + } + + bool issue1742() { + TEST_START(); + auto json = R"( { + "code": 0, + "method": "subscribe", + "result": { + "instrument_name": "DAI_USDC", + "subscription": "trade.DAI_USDC", + "channel": "trade", + "data": [ + [1,2,3,4] + ] + } +} )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + simdjson::singlestage::array data; + ASSERT_SUCCESS(doc["result"]["data"].get_array().get(data)); + for (auto d : data) { + simdjson::singlestage::array arr; + ASSERT_SUCCESS(d.get_array().get(arr)); + size_t count{}; + ASSERT_SUCCESS(arr.count_elements().get(count)); + ASSERT_EQUAL(count, 4); + } + TEST_SUCCEED(); + } + + bool value_to_array(simdjson::singlestage::value val) { + singlestage::json_type t; + ASSERT_SUCCESS(val.type().get(t)); + ASSERT_EQUAL(t, singlestage::json_type::array); + simdjson::singlestage::array arr; + ASSERT_SUCCESS(val.get_array().get(arr)); + if(count_empty(arr) != true) { return false; } + return true; + } + bool empty_rewind_convoluted() { + TEST_START(); + const auto json = R"( [] )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::value val; + ASSERT_SUCCESS(doc.get_value().get(val)); + if(!value_to_array(val)) { return false; } + TEST_SUCCEED(); + } +#if SIMDJSON_EXCEPTIONS + bool value_to_array_except(simdjson::singlestage::value val) { + singlestage::json_type t = val.type(); + ASSERT_EQUAL(t, singlestage::json_type::array); + if(count_empty(simdjson::singlestage::array(val)) != true) { return false; } + return true; + } + bool empty_rewind_convoluted_with_exceptions() { + TEST_START(); + const auto json = R"( [] )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::value val; + ASSERT_SUCCESS(doc.get_value().get(val)); + if(value_to_array_except(val) != true) { return false; } + TEST_SUCCEED(); + } +#endif + bool iterate_array() { + TEST_START(); + const auto json = R"( [ [ 1, 10, 100 ] ] )"_padded; + const uint64_t expected_value[] = { 1, 10, 100 }; + + SUBTEST("singlestage::array", test_singlestage_doc(json, [&](auto doc_result) { + bool found = false; + for (simdjson_result array_result : doc_result) { + ASSERT_TRUE(!found); found = true; + + singlestage::array array; + ASSERT_RESULT( array_result.type(), json_type::array ); + ASSERT_SUCCESS( array_result.get(array) ); + + size_t i=0; + for (auto value : array) { + int64_t actual{}; + ASSERT_SUCCESS( value.get(actual) ); + ASSERT_EQUAL(actual, expected_value[i]); + i++; + } + ASSERT_EQUAL(i*sizeof(uint64_t), sizeof(expected_value)); + } + ASSERT_TRUE(found); + return true; + })); + + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + bool found = false; + for (simdjson_result array_result : doc_result) { + ASSERT_TRUE(!found); found = true; + + singlestage::array array; + ASSERT_RESULT( array_result.type(), json_type::array ); + ASSERT_SUCCESS( array_result.get(array) ); + + size_t i=0; + for (simdjson_unused auto value : array) { int64_t actual; ASSERT_SUCCESS( value.get(actual) ); ASSERT_EQUAL(actual, expected_value[i]); i++; } + ASSERT_EQUAL(i*sizeof(uint64_t), sizeof(expected_value)); + } + ASSERT_TRUE(found); + return true; + })); + + SUBTEST("singlestage::value", test_singlestage_doc(json, [&](auto doc_result) { + bool found = false; + for (simdjson_result array_result : doc_result) { + ASSERT_TRUE(!found); found = true; + + singlestage::value array; + ASSERT_SUCCESS( std::move(array_result).get(array) ); + ASSERT_RESULT( array.type(), json_type::array ); + size_t i=0; + for (simdjson_unused auto value : array) { int64_t actual; ASSERT_SUCCESS( value.get(actual) ); ASSERT_EQUAL(actual, expected_value[i]); i++; } + ASSERT_EQUAL(i*sizeof(uint64_t), sizeof(expected_value)); + } + ASSERT_TRUE(found); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + bool found = false; + for (simdjson_result array_result : doc_result) { + ASSERT_TRUE(!found); found = true; + + size_t i=0; + ASSERT_RESULT( array_result.type(), json_type::array ); + for (simdjson_unused auto value : array_result) { int64_t actual; ASSERT_SUCCESS( value.get(actual) ); ASSERT_EQUAL(actual, expected_value[i]); i++; } + ASSERT_EQUAL(i*sizeof(uint64_t), sizeof(expected_value)); + } + ASSERT_TRUE(found); + return true; + })); + TEST_SUCCEED(); + } + + bool iterate_array_partial_children() { + TEST_START(); + auto json = R"( + [ + 0, + [], + {}, + { "x": 3, "y": 33 }, + { "x": 4, "y": 44 }, + { "x": 5, "y": 55 }, + { "x": 6, "y": 66 }, + [ 7, 77, 777 ], + [ 8, 88, 888 ], + { "a": [ { "b": [ 9, 99 ], "c": 999 }, 9999 ], "d": 99999 }, + 10 + ] + )"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + size_t i = 0; + for (auto value : doc_result) { + ASSERT_SUCCESS(value); + + switch (i) { + case 0: { + std::cout << " - After ignoring empty scalar ..." << std::endl; + break; + } + case 1: { + std::cout << " - After ignoring empty array ..." << std::endl; + break; + } + case 2: { + std::cout << " - After ignoring empty object ..." << std::endl; + break; + } + // Break after using first value in child object + case 3: { + for (auto child_field : value.get_object()) { + singlestage::raw_json_string k; + ASSERT_SUCCESS( child_field.key().get(k) ); + ASSERT_EQUAL(k, "x"); + uint64_t x; + ASSERT_SUCCESS( child_field.value().get(x) ); + ASSERT_EQUAL(x, 3); + break; // Break after the first value + } + std::cout << " - After using first value in child object ..." << std::endl; + break; + } + + // Break without using first value in child object + case 4: { + for (auto child_field : value.get_object()) { + singlestage::raw_json_string k; + ASSERT_SUCCESS( child_field.key().get(k) ); + ASSERT_EQUAL(k, "x"); + break; + } + std::cout << " - After reaching (but not using) first value in child object ..." << std::endl; + break; + } + + // Only look up one field in child object + case 5: { + uint64_t x; + ASSERT_SUCCESS( value["x"].get(x) ); + ASSERT_EQUAL( x, 5 ); + std::cout << " - After looking up one field in child object ..." << std::endl; + break; + } + + // Only look up one field in child object, but don't use it + case 6: { + ASSERT_SUCCESS( value["x"] ); + std::cout << " - After looking up (but not using) one field in child object ..." << std::endl; + break; + } + + // Break after first value in child array + case 7: { + for (auto child_value : value) { + uint64_t x; + ASSERT_SUCCESS( child_value.get(x) ); + ASSERT_EQUAL( x, 7 ); + break; + } + std::cout << " - After using first value in child array ..." << std::endl; + break; + } + + // Break without using first value in child array + case 8: { + for (auto child_value : value) { + ASSERT_SUCCESS(child_value); + break; + } + std::cout << " - After reaching (but not using) first value in child array ..." << std::endl; + break; + } + + // Break out of multiple child loops + case 9: { + for (auto child1 : value.get_object()) { + for (auto child2 : child1.value().get_array()) { + for (auto child3 : child2.get_object()) { + for (auto child4 : child3.value().get_array()) { + uint64_t x; + ASSERT_SUCCESS( child4.get(x) ); + ASSERT_EQUAL( x, 9 ); + break; + } + break; + } + break; + } + break; + } + std::cout << " - After breaking out of quadruply-nested arrays and objects ..." << std::endl; + break; + } + + // Test the actual value + case 10: { + uint64_t actual_value; + ASSERT_SUCCESS( value.get(actual_value) ); + ASSERT_EQUAL( actual_value, 10 ); + break; + } + } + + i++; + } + ASSERT_EQUAL( i, 11 ); // Make sure we found all the keys we expected + return true; + })); + return true; + } + + bool iterate_empty_array() { + TEST_START(); + auto json = "[]"_padded; + SUBTEST("singlestage::array", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::array array; + ASSERT_SUCCESS( doc_result.get(array) ); + for (simdjson_unused auto value : array) { TEST_FAIL("Unexpected value"); } + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + simdjson_result array_result = doc_result.get_array(); + for (simdjson_unused auto value : array_result) { TEST_FAIL("Unexpected value"); } + return true; + })); + SUBTEST("singlestage::document", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::document doc; + ASSERT_SUCCESS( std::move(doc_result).get(doc) ); + for (simdjson_unused auto value : doc) { TEST_FAIL("Unexpected value"); } + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + for (simdjson_unused auto value : doc_result) { TEST_FAIL("Unexpected value"); } + return true; + })); + SUBTEST("singlestage::array-document-rewind", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::array array; + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_SUCCESS( doc_result.get(array) ); + + size_t i = 0; + for (auto value : array) { (void) value; i++; } + ASSERT_EQUAL(i, 0); + + doc_result.rewind(); + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_SUCCESS( doc_result.get(array) ); + i = 0; + for (auto value : array) { (void) value; i++; } + ASSERT_EQUAL(i, 0); + return true; + })); + SUBTEST("singlestage::array-rewind", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::array array; + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_SUCCESS( doc_result.get(array) ); + + size_t i = 0; + for (auto value : array) { (void) value; i++; } + ASSERT_EQUAL(i, 0); + + array.reset(); + i = 0; + for (auto value : array) { (void) value; i++; } + ASSERT_EQUAL(i, 0); + return true; + })); + TEST_SUCCEED(); + } + +#if SIMDJSON_EXCEPTIONS + + bool iterate_array_exception() { + TEST_START(); + auto json = R"([ 1, 10, 100 ])"_padded; + const uint64_t expected_value[] = { 1, 10, 100 }; + + ASSERT_TRUE(test_singlestage_doc(json, [&](auto doc_result) { + size_t i=0; + for (int64_t actual : doc_result) { ASSERT_EQUAL(actual, expected_value[i]); i++; } + ASSERT_EQUAL(i*sizeof(uint64_t), sizeof(expected_value)); + return true; + })); + TEST_SUCCEED(); + } + + bool iterate_empty_object_exception() { + TEST_START(); + auto json = R"({})"_padded; + + ASSERT_TRUE(test_singlestage_doc(json, [&](auto doc_result) { + for (simdjson_unused singlestage::field field : doc_result.get_object()) { + TEST_FAIL("Unexpected field"); + } + return true; + })); + + TEST_SUCCEED(); + } + + bool iterate_empty_array_exception() { + TEST_START(); + auto json = "[]"_padded; + + ASSERT_TRUE(test_singlestage_doc(json, [&](auto doc_result) { + for (simdjson_unused singlestage::value value : doc_result) { TEST_FAIL("Unexpected value"); } + return true; + })); + + TEST_SUCCEED(); + } + +#endif // SIMDJSON_EXCEPTIONS + + bool run() { + return + issue1977() && + issue1876() && + issue1742() && + empty_rewind_convoluted() && + empty_rewind() && + iterate_empty_array_count() && + iterate_sub_array_count() && + iterate_complex_array_count() && + iterate_bad_array_count() && + iterate_array_count() && + issue1588() && + iterate_array() && + iterate_document_array_count() && + iterate_bad_document_array_count() && + iterate_document_array() && + iterate_empty_array() && + iterate_array_partial_children() && +#if SIMDJSON_EXCEPTIONS + empty_rewind_convoluted_with_exceptions() && + iterate_array_exception() && +#endif // SIMDJSON_EXCEPTIONS + true; + } + +} // namespace dom_api_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, array_tests::run); +} diff --git a/tests/singlestage/singlestage_assert_out_of_order_values.cpp b/tests/singlestage/singlestage_assert_out_of_order_values.cpp new file mode 100644 index 000000000..8e3c18e2b --- /dev/null +++ b/tests/singlestage/singlestage_assert_out_of_order_values.cpp @@ -0,0 +1,80 @@ +#include +#include +#include +#include +#include +#include "simdjson.h" + + +// We get spurious "maybe used uninitialized" warnings under GCC 12. +using namespace simdjson; +#if defined(__GNUC__) && !defined(__clang__) +#if __GNUC__ >= 12 +SIMDJSON_PUSH_DISABLE_ALL_WARNINGS +#endif +#endif + +// This ensures the compiler can't rearrange them into the proper order (which causes it to work!) +simdjson_never_inline bool check_point(simdjson_result xval, simdjson_result yval) { + // Verify the expected release behavior + uint64_t x, y; + if (!xval.get(x)) { return false; } + if (!yval.get(y)) { return false; } + std::cout << x << "," << y << std::endl; + return true; +} + +bool test_check_point() { + auto json = R"( + { + "x": 1, + "y": 2 3 + )"_padded; + singlestage::parser parser; + auto doc = parser.iterate(json); + return check_point(doc["x"], doc["y"]); +} + +// Run a test function in a fork and check whether it exited with an assert signal +template +bool assert_test(const F& f) { + pid_t pid = fork(); + if (pid == -1) { + std::cerr << "fork failed" << std::endl; + exit(1); + } + + if (!pid) { + // + // This code runs in the fork (so we run the test function) + // + bool succeeded = f(); + exit(succeeded ? 0 : 1); + } + + // + // This code runs in the original executable (so we wait for the fork and check the exit code) + // + int exit_code = 0; + if (waitpid(pid, &exit_code, 0) == pid_t(-1)) { + std::cerr << "waitpid failed: " << std::string_view(strerror(errno)) << std::endl; + exit(1); + } + + // Test passes if the child exited with an assert signal + return WIFSIGNALED(exit_code); +} + +int main(void) { + + // To verify that the program asserts (which sends a signal), we fork a new process and use wait() + // to check the resulting error code. If it's a signal error code, we consider the + // test to have passed. + // From https://stackoverflow.com/a/33694733 + + bool succeeded = true; +#ifndef NDEBUG + succeeded |= assert_test(test_check_point); +#endif + return succeeded ? 0 : 1; +} diff --git a/tests/singlestage/singlestage_compilation_tests.cpp b/tests/singlestage/singlestage_compilation_tests.cpp new file mode 100644 index 000000000..e7e69dec3 --- /dev/null +++ b/tests/singlestage/singlestage_compilation_tests.cpp @@ -0,0 +1,66 @@ +#include +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +#if SIMDJSON_EXCEPTIONS + +// bogus functions for compilation tests +void process1(int ) {} +void process2(int ) {} +void process3(int ) {} + +// Do not run this, it is only meant to compile +void compilation_test_1() { + const padded_string bogus = ""_padded; + singlestage::parser parser; + auto doc = parser.iterate(bogus); + for (singlestage::object my_object : doc["mykey"]) { + for (auto field : my_object) { + if (field.key() == "key_value1") { process1(field.value()); } + else if (field.key() == "key_value2") { process2(field.value()); } + else if (field.key() == "key_value3") { process3(field.value()); } + } + } +} + +// Do not run this, it is only meant to compile + void compilation_test_2() { + const padded_string bogus = ""_padded; + singlestage::parser parser; + auto doc = parser.iterate(bogus); + std::set default_users; + singlestage::array tweets = doc["statuses"].get_array(); + for (auto tweet_value : tweets) { + auto tweet = tweet_value.get_object(); + singlestage::object user = tweet["user"].get_object(); + std::string_view screen_name = user["screen_name"].get_string(); + bool default_profile = user["default_profile"].get_bool(); + if (default_profile) { default_users.insert(screen_name); } + } +} + + +// Do not run this, it is only meant to compile +void compilation_test_3() { + const padded_string bogus = ""_padded; + singlestage::parser parser; + auto doc = parser.iterate(bogus); + singlestage::array tweets; + if(! doc["statuses"].get(tweets)) { return; } + for (auto tweet_value : tweets) { + auto tweet = tweet_value.get_object(); + for (auto field : tweet) { + std::string_view key = field.unescaped_key().value(); + std::cout << "key = " << key << std::endl; + std::string_view val = std::string_view(field.value()); + std::cout << "value (assuming it is a string) = " << val << std::endl; + } + } +} +#endif // SIMDJSON_EXCEPTIONS + +int main(void) { + return 0; +} diff --git a/tests/singlestage/singlestage_document_stream_tests.cpp b/tests/singlestage/singlestage_document_stream_tests.cpp new file mode 100644 index 000000000..51df9c2ea --- /dev/null +++ b/tests/singlestage/singlestage_document_stream_tests.cpp @@ -0,0 +1,844 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace document_stream_tests { + + template + bool process_doc(T &docref) { + int64_t val{}; + ASSERT_SUCCESS(docref.at_pointer("/4").get(val)); + ASSERT_EQUAL(val, 5); + return true; + } + bool truncated_utf8() { + TEST_START(); + // truncated UTF-8 + auto json = "\"document1\" \xC3"_padded; + + singlestage::parser parser; + singlestage::document_stream stream; + auto oderror = parser.iterate_many(json).get(stream); + if (oderror) { + std::cerr << "singlestage iterate_many error: " << oderror << std::endl; + return false; + } + int document_index = 0; + auto i = stream.begin(); + for (; i != stream.end(); ++i) { + singlestage::document_reference doc; + auto err = (*i).get(doc); + document_index++; + // With the fallback routine, we might detect a 'document' since + // it does not do UTF-8 validation in stage one, but it will do + // so when we access the document. + if(err == SUCCESS) { + singlestage::json_type t; + err = doc.type().get(t); + } + if((err == SUCCESS) && (document_index != 1)) { + std::cerr << "Only the first document should be valid." << std::endl; + return false; + } + if((err != SUCCESS) && (document_index != 2)) { + std::cerr << "singlestage iterate_many error: " << err << std::endl; + return false; + } + } + if(document_index < 1) { + std::cerr << "singlestage should have accessed at least one document" << std::endl; + return false; + } + TEST_SUCCEED(); + } + bool issue1729() { + // This example is not a good application of iterate_many since there is + // a single JSON document. + TEST_START(); + auto json = R"({ + "t": 5649, + "ng": 5, + "lu": 4086, + "g": [{ + "t": "Temps", + "xvy": 0, + "pd": 60, + "sz": 3, + "l": [ + "Low Temp", + "High Temp", + "Smooth Avg" + ], + "c": [ + "green", + "orange", + "cyan" + ], + "d": [ + 80.48750305, + 80.82499694, + 80.65625 + ] + }, + { + "t": "Pump Status", + "xvy": 0, + "pd": 60, + "sz": 3, + "l": [ + "Pump", + "Thermal", + "Light" + ], + "c": [ + "green", + "orange", + "cyan" + ], + "d": [ + 0, + 0, + 0 + ] + }, + { + "t": "Lux", + "xvy": 0, + "pd": 60, + "sz": 4, + "l": [ + "Value", + "Smooth", + "Low", + "High" + ], + "c": [ + "green", + "orange", + "cyan", + "yellow" + ], + "d": [ + 2274.62939453, + 2277.45947265, + 4050, + 4500 + ] + }, + { + "t": "Temp Diff", + "xvy": 0, + "pd": 60, + "sz": 4, + "l": [ + "dFarenheit", + "sum", + "Low", + "High" + ], + "c": [ + "green", + "orange", + "cyan", + "yellow" + ], + "d": [ + 0, + 0, + 0.5, + 10 + ] + }, + { + "t": "Power", + "xvy": 0, + "pd": 60, + "sz": 3, + "l": [ + "watts (est. light) ", + "watts (1.9~gpm) ", + "watts (1gpm) " + ], + "c": [ + "green", + "orange", + "cyan" + ], + "d": [ + 181.78063964, + 114.88922882, + 59.35943603 + ] + } + ] +})"_padded; + + singlestage::parser parser; + singlestage::document_stream stream; + auto oderror = parser.iterate_many(json).get(stream); + if (oderror) { + std::cerr << "singlestage iterate_many error: " << oderror << std::endl; + return false; + } + auto i = stream.begin(); + for (; i != stream.end(); ++i) { + singlestage::document_reference doc; + auto err = (*i).get(doc); + if(err != SUCCESS) { + std::cerr << "singlestage iterate_many error: " << err << std::endl; + return false; + } + } + TEST_SUCCEED(); + } + + bool issue1977() { + TEST_START(); + std::string json = R"( 1111 })"; + singlestage::parser odparser; + singlestage::document_stream odstream; + ASSERT_SUCCESS(odparser.iterate_many(json).get(odstream)); + + auto i = odstream.begin(); + for (; i != odstream.end(); ++i) { + ASSERT_TRUE(false); + } + ASSERT_TRUE(i.current_index() == 0); + TEST_SUCCEED(); + } + + bool issue1683() { + TEST_START(); + std::string json = R"([1,2,3,4,5] +[1,2,3,4,5] +[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100] +[1,2,3,4,5])"; + + singlestage::parser odparser; + singlestage::document_stream odstream; + + // iterate_many all at once + auto oderror = odparser.iterate_many(json, 50).get(odstream); + if (oderror) { std::cerr << "singlestage iterate_many error: " << oderror << std::endl; return false; } + + size_t currindex = 0; + auto i = odstream.begin(); + for (; i != odstream.end(); ++i) { + singlestage::document_reference doc; + auto err = (*i).get(doc); + if(err == SUCCESS) { if(!process_doc(doc)) {return false; } } + currindex = i.current_index(); + if (err == simdjson::CAPACITY) { + ASSERT_EQUAL(currindex, 24); + ASSERT_EQUAL(odstream.truncated_bytes(), 305); + break; + } else if (err) { + TEST_FAIL(std::string("singlestage: error accessing jsonpointer: ") + simdjson::error_message(err)); + } + } + ASSERT_EQUAL(odstream.truncated_bytes(), 305); + + // iterate line-by-line + std::stringstream ss(json); + std::string oneline; + oneline.reserve(json.size() + SIMDJSON_PADDING); + while (getline(ss, oneline)) { + singlestage::document doc; + ASSERT_SUCCESS(odparser.iterate(oneline).get(doc)); + if( ! process_doc(doc) ) { return false; } + } + TEST_SUCCEED(); + } + + bool simple_document_iteration() { + TEST_START(); + auto json = R"([1,[1,2]] {"a":1,"b":2} {"o":{"1":1,"2":2}} [1,2,3])"_padded; + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(json).get(stream)); + std::string_view expected[4] = {"[1,[1,2]]", "{\"a\":1,\"b\":2}", "{\"o\":{\"1\":1,\"2\":2}}", "[1,2,3]"}; + size_t counter{0}; + for(auto doc : stream) { + ASSERT_TRUE(counter < 4); + std::string_view view; + ASSERT_SUCCESS(to_json_string(doc).get(view)); + ASSERT_EQUAL(view, expected[counter++]); + } + ASSERT_EQUAL(counter, 4); + TEST_SUCCEED(); + } + + bool simple_document_iteration_multiple_batches() { + TEST_START(); + auto json = R"([1,[1,2]] {"a":1,"b":2} {"o":{"1":1,"2":2}} [1,2,3])"_padded; + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(json,32).get(stream)); + std::string_view expected[4] = {"[1,[1,2]]", "{\"a\":1,\"b\":2}", "{\"o\":{\"1\":1,\"2\":2}}", "[1,2,3]"}; + size_t counter{0}; + for(auto i = stream.begin(); i != stream.end(); ++i) { + ASSERT_TRUE(counter < 4); + ASSERT_EQUAL(i.source(), expected[counter++]); + } + ASSERT_EQUAL(counter, 4); + TEST_SUCCEED(); + } + + bool simple_document_iteration_with_parsing() { + TEST_START(); + auto json = R"([1,[1,2]] {"a":1,"b":2} {"o":{"1":1,"2":2}} [1,2,3])"_padded; + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(json).get(stream)); + std::string_view expected[4] = {"[1,[1,2]]", "{\"a\":1,\"b\":2}", "{\"o\":{\"1\":1,\"2\":2}}", "[1,2,3]"}; + size_t counter{0}; + auto i = stream.begin(); + int64_t x; + + ASSERT_EQUAL(i.source(),expected[counter++]); + ASSERT_SUCCESS( (*i).at_pointer("/1/1").get(x) ); + ASSERT_EQUAL(x,2); + ++i; + + ASSERT_EQUAL(i.source(),expected[counter++]); + simdjson_result xxx = *i; + ASSERT_SUCCESS( xxx.find_field("a").get(x) ); + ASSERT_EQUAL(x,1); + ++i; + + ASSERT_EQUAL(i.source(),expected[counter++]); + ASSERT_SUCCESS( (*i).at_pointer("/o/2").get(x) ); + ASSERT_EQUAL(x,2); + ++i; + + ASSERT_EQUAL(i.source(),expected[counter++]); + ASSERT_SUCCESS( (*i).at_pointer("/2").get(x) ); + ASSERT_EQUAL(x,3); + ++i; + + if (i != stream.end()) { return false; } + + TEST_SUCCEED(); + } + + bool atoms_json() { + TEST_START(); + auto json = R"(5 true 20.3 "string" )"_padded; + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(json).get(stream)); + + std::string_view expected[4] = {"5", "true", "20.3", "\"string\""}; + size_t counter{0}; + for (auto i = stream.begin(); i != stream.end(); ++i) { + ASSERT_EQUAL(i.source(), expected[counter++]); + } + ASSERT_EQUAL(counter,4); + TEST_SUCCEED(); + } + + bool doc_index() { + TEST_START(); + auto json = R"({"z":5} {"1":1,"2":2,"4":4} [7, 10, 9] [15, 11, 12, 13] [154, 110, 112, 1311])"_padded; + std::string_view expected[5] = {R"({"z":5})",R"({"1":1,"2":2,"4":4})","[7, 10, 9]","[15, 11, 12, 13]","[154, 110, 112, 1311]"}; + size_t expected_indexes[5] = {0, 9, 29, 44, 65}; + + singlestage::parser parser; + singlestage::document_stream stream; + size_t counter{0}; + ASSERT_SUCCESS(parser.iterate_many(json).get(stream)); + for(auto i = stream.begin(); i != stream.end(); ++i) { + ASSERT_TRUE(counter < 5); + ASSERT_EQUAL(i.current_index(), expected_indexes[counter]); + ASSERT_EQUAL(i.source(), expected[counter]); + counter++; + } + ASSERT_EQUAL(counter, 5); + TEST_SUCCEED(); + } + + bool doc_index_multiple_batches() { + TEST_START(); + auto json = R"({"z":5} {"1":1,"2":2,"4":4} [7, 10, 9] [15, 11, 12, 13] [154, 110, 112, 1311])"_padded; + std::string_view expected[5] = {R"({"z":5})",R"({"1":1,"2":2,"4":4})","[7, 10, 9]","[15, 11, 12, 13]","[154, 110, 112, 1311]"}; + size_t expected_indexes[5] = {0, 9, 29, 44, 65}; + + singlestage::parser parser; + singlestage::document_stream stream; + size_t counter{0}; + ASSERT_SUCCESS(parser.iterate_many(json,32).get(stream)); + for(auto i = stream.begin(); i != stream.end(); ++i) { + ASSERT_TRUE(counter < 5); + ASSERT_EQUAL(i.current_index(), expected_indexes[counter]); + ASSERT_EQUAL(i.source(), expected[counter]); + counter++; + } + ASSERT_EQUAL(counter, 5); + TEST_SUCCEED(); + } + + bool source_test() { + TEST_START(); + auto json = R"([1,[1,2]] {"a":1,"b":2} {"o":{"1":1,"2":2}} [1,2,3] )"_padded; + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(json).get(stream)); + std::string_view expected[4] = {"[1,[1,2]]", "{\"a\":1,\"b\":2}", "{\"o\":{\"1\":1,\"2\":2}}", "[1,2,3]"}; + size_t counter{0}; + for (auto i = stream.begin(); i != stream.end(); ++i) { + ASSERT_EQUAL(i.source(), expected[counter++]); + } + ASSERT_EQUAL(counter,4); + TEST_SUCCEED(); + } + + bool truncated() { + TEST_START(); + // The last JSON document is intentionally truncated. + auto json = R"([1,2,3] {"1":1,"2":3,"4":4} [1,2 )"_padded; + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(json).get(stream)); + + size_t counter{0}; + for (auto i = stream.begin(); i != stream.end(); ++i) { + counter++; + } + size_t truncated = stream.truncated_bytes(); + ASSERT_EQUAL(truncated, 6); + ASSERT_EQUAL(counter,2); + TEST_SUCCEED(); + } + + bool truncated_complete_docs() { + TEST_START(); + auto json = R"([1,2,3] {"1":1,"2":3,"4":4} [1,2] )"_padded; + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(json).get(stream)); + + size_t counter{0}; + for (auto i = stream.begin(); i != stream.end(); ++i) { + counter++; + } + size_t truncated = stream.truncated_bytes(); + ASSERT_EQUAL(truncated, 0); + ASSERT_EQUAL(counter,3); + TEST_SUCCEED(); + } + + bool truncated_unclosed_string() { + TEST_START(); + // The last JSON document is intentionally truncated. + auto json = R"([1,2,3] {"1":1,"2":3,"4":4} "intentionally unclosed string )"_padded; + singlestage::parser parser; + singlestage::document_stream stream; + // We use a window of json.size() though any large value would do. + ASSERT_SUCCESS( parser.iterate_many(json).get(stream) ); + size_t counter{0}; + for(auto i = stream.begin(); i != stream.end(); ++i) { + counter++; + } + size_t truncated = stream.truncated_bytes(); + ASSERT_EQUAL(counter,2); + ASSERT_EQUAL(truncated,32); + TEST_SUCCEED(); + } + + bool truncated_unclosed_string_in_object() { + // The last JSON document is intentionally truncated. + auto json = R"([1,2,3] {"1":1,"2":3,"4":4} {"key":"intentionally unclosed string )"_padded; + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS( parser.iterate_many(json).get(stream) ); + size_t counter{0}; + for(auto i = stream.begin(); i != stream.end(); ++i) { + counter++; + } + size_t truncated = stream.truncated_bytes(); + ASSERT_EQUAL(counter,2); + ASSERT_EQUAL(truncated,39); + TEST_SUCCEED(); + } + + bool small_window() { + TEST_START(); + std::vector input; + input.push_back('['); + for(size_t i = 1; i < 1024; i++) { + input.push_back('1'); + input.push_back(i < 1023 ? ',' : ']'); + } + auto json = simdjson::padded_string(input.data(),input.size()); + singlestage::parser parser; + size_t window_size{1024}; // deliberately too small + singlestage::document_stream stream; + ASSERT_SUCCESS( parser.iterate_many(json, window_size).get(stream) ); + auto i = stream.begin(); + ASSERT_ERROR(i.error(), CAPACITY); + ASSERT_EQUAL(stream.truncated_bytes(), json.length()); + TEST_SUCCEED(); + } + + bool large_window() { + TEST_START(); + #if SIZE_MAX > 17179869184 + auto json = R"({"error":[],"result":{"token":"xxx"}}{"error":[],"result":{"token":"xxx"}})"_padded; + singlestage::parser parser; + uint64_t window_size{17179869184}; // deliberately too big + singlestage::document_stream stream; + ASSERT_SUCCESS( parser.iterate_many(json, size_t(window_size)).get(stream) ); + auto i = stream.begin(); + ASSERT_ERROR(i.error(),CAPACITY); + #endif + TEST_SUCCEED(); + } + + bool test_leading_spaces() { + TEST_START(); + auto input = R"( [1,1] [1,2] [1,3] [1,4] [1,5] [1,6] [1,7] [1,8] [1,9] [1,10] [1,11] [1,12] [1,13] [1,14] [1,15] )"_padded;; + size_t count{0}; + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(input, 32).get(stream)); + for(auto i = stream.begin(); i != stream.end(); ++i) { + ASSERT_SUCCESS(i.error()); + count++; + } + ASSERT_EQUAL(count,15); + TEST_SUCCEED(); + } + + + bool test_crazy_leading_spaces() { + TEST_START(); + auto input = R"( [1,1] [1,2] [1,3] [1,4] [1,5] [1,6] [1,7] [1,8] [1,9] [1,10] [1,11] [1,12] [1,13] [1,14] [1,15] )"_padded;; + size_t count{0}; + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(input, 32).get(stream)); + for(auto i = stream.begin(); i != stream.end(); ++i) { + ASSERT_SUCCESS(i.error()); + count++; + } + ASSERT_EQUAL(count,15); + TEST_SUCCEED(); + } + + bool adversarial_single_document() { + TEST_START(); + auto json = R"({"f[)"_padded; + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(json).get(stream)); + size_t count{0}; + for (auto doc : stream) { + (void)doc; + count++; + } + ASSERT_EQUAL(count,0); + TEST_SUCCEED(); + } + + bool adversarial_single_document_array() { + TEST_START(); + auto json = R"(["this is an unclosed string ])"_padded; + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(json).get(stream)); + size_t count{0}; + for (auto doc : stream) { + (void)doc; + count++; + } + ASSERT_EQUAL(count,0); + TEST_SUCCEED(); + } + + bool document_stream_test() { + TEST_START(); + fflush(NULL); + const size_t n_records = 100; + std::string data; + std::vector buf(1024); + // Generating data + for (size_t i = 0; i < n_records; ++i) { + size_t n = snprintf(buf.data(), + buf.size(), + "{\"id\": %zu, \"name\": \"name%zu\", \"gender\": \"%s\", " + "\"ete\": {\"id\": %zu, \"name\": \"eventail%zu\"}}", + i, i, (i % 2) ? "homme" : "femme", i % 10, i % 10); + if (n >= buf.size()) { abort(); } + data += std::string(buf.data(), n); + } + + for(size_t batch_size = 1000; batch_size < 2000; batch_size += (batch_size>1050?10:1)) { + fflush(NULL); + simdjson::padded_string str(data); + singlestage::parser parser; + singlestage::document_stream stream; + size_t count{0}; + ASSERT_SUCCESS( parser.iterate_many(str, batch_size).get(stream) ); + for (auto doc : stream) { + int64_t keyid{}; + ASSERT_SUCCESS( doc["id"].get(keyid) ); + ASSERT_EQUAL( keyid, int64_t(count) ); + + count++; + } + ASSERT_EQUAL(count,n_records); + } + TEST_SUCCEED(); + } + + + bool issue1668() { + TEST_START(); + auto json = R"([1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100])"_padded; + + singlestage::parser odparser; + singlestage::document_stream odstream; + ASSERT_SUCCESS( odparser.iterate_many(json.data(), json.length(), 50).get(odstream) ); + for (auto doc: odstream) { + singlestage::value val; + ASSERT_ERROR(doc.at_pointer("/40").get(val), CAPACITY); + ASSERT_EQUAL(odstream.truncated_bytes(), json.length()); + } + TEST_SUCCEED(); + } + + bool issue1668_long() { + TEST_START(); + auto json = R"([1,2,3,4,5] [1,2,3,4,5] [1,2,3,4,5] [1,2,3,4,5] [1,2,3,4,5] [1,2,3,4,5] [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100])"_padded; + + singlestage::parser odparser; + singlestage::document_stream odstream; + size_t counter{0}; + ASSERT_SUCCESS( odparser.iterate_many(json.data(), json.length(), 50).get(odstream) ); + for (auto doc: odstream) { + if(counter < 6) { + int64_t val{}; + ASSERT_SUCCESS(doc.at_pointer("/4").get(val)); + ASSERT_EQUAL(val, 5); + } else { + singlestage::value val; + ASSERT_ERROR(doc.at_pointer("/4").get(val), CAPACITY); + // We left 293 bytes unprocessed. + ASSERT_EQUAL(odstream.truncated_bytes(), 293); + } + counter++; + } + TEST_SUCCEED(); + } + + bool document_stream_utf8_test() { + TEST_START(); + fflush(NULL); + const size_t n_records = 100; + std::string data; + std::vector buf(1024); + // Generating data + for (size_t i = 0; i < n_records; ++i) { + size_t n = snprintf(buf.data(), + buf.size(), + "{\"id\": %zu, \"name\": \"name%zu\", \"gender\": \"%s\", " + "\"\xC3\xA9t\xC3\xA9\": {\"id\": %zu, \"name\": \"\xC3\xA9ventail%zu\"}}", + i, i, (i % 2) ? "\xE2\xBA\x83" : "\xE2\xBA\x95", i % 10, i % 10); + if (n >= buf.size()) { abort(); } + data += std::string(buf.data(), n); + } + + for(size_t batch_size = 1000; batch_size < 2000; batch_size += (batch_size>1050?10:1)) { + fflush(NULL); + simdjson::padded_string str(data); + singlestage::parser parser; + singlestage::document_stream stream; + size_t count{0}; + ASSERT_SUCCESS( parser.iterate_many(str, batch_size).get(stream) ); + for (auto doc : stream) { + int64_t keyid{}; + ASSERT_SUCCESS( doc["id"].get(keyid) ); + ASSERT_EQUAL( keyid, int64_t(count) ); + + count++; + } + ASSERT_EQUAL( count, n_records ) + } + TEST_SUCCEED(); + } + + bool stress_data_race() { + TEST_START(); + // Correct JSON. + auto input = R"([1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] )"_padded;; + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(input, 32).get(stream)); + for(auto i = stream.begin(); i != stream.end(); ++i) { + ASSERT_SUCCESS(i.error()); + } + TEST_SUCCEED(); + } + + bool stress_data_race_with_error() { + TEST_START(); + #if SIMDJSON_THREAD_ENABLED + std::cout << "ENABLED" << std::endl; + #endif + // Intentionally broken + auto input = R"([1,23] [1,23] [1,23] [1,23 [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] )"_padded; + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(input, 32).get(stream)); + size_t count{0}; + for(auto i = stream.begin(); i != stream.end(); ++i) { + auto error = i.error(); + if(count <= 3) { + ASSERT_SUCCESS(error); + } else { + ASSERT_ERROR(error,TAPE_ERROR); + break; + } + count++; + } + TEST_SUCCEED(); + } + + bool fuzzaccess() { + TEST_START(); + // Issue 38801 in oss-fuzz + auto json = "\xff \n~~\n{}"_padded; + singlestage::parser parser; + singlestage::document_stream docs; + ASSERT_SUCCESS(parser.iterate_many(json).get(docs)); + size_t bool_count = 0; + size_t total_count = 0; + for (auto doc : docs) { + total_count++; + bool b; + if(doc.get_bool().get(b) == SUCCESS) { bool_count +=b; } + // If we don't access the document at all, other than get_bool(), + // then some simdjson kernels will allow you to iterate through + // 3 'documents'. By asking for the type, we make the iteration + // terminate after the first 'document'. + singlestage::json_type t; + if(doc.type().get(t) != SUCCESS) { break; } + } + return (bool_count == 0) && (total_count == 1); + } + + bool string_with_trailing() { + TEST_START(); + auto json = R"( "a string" badstuff )"_padded; + singlestage::parser parser; + singlestage::document_stream doc; + ASSERT_SUCCESS(parser.iterate_many(json).get(doc)); + std::string_view view; + ASSERT_SUCCESS((*doc.begin()).get_string().get(view)); + ASSERT_EQUAL(view,"a string"); + TEST_SUCCEED(); + } + + bool uint64_with_trailing() { + TEST_START(); + auto json = R"( 133 badstuff )"_padded; + singlestage::parser parser; + singlestage::document_stream doc; + ASSERT_SUCCESS(parser.iterate_many(json).get(doc)); + uint64_t x; + ASSERT_SUCCESS((*doc.begin()).get_uint64().get(x)); + ASSERT_EQUAL(x,133); + TEST_SUCCEED(); + } + + bool int64_with_trailing() { + TEST_START(); + auto json = R"( 133 badstuff )"_padded; + singlestage::parser parser; + singlestage::document_stream doc; + ASSERT_SUCCESS(parser.iterate_many(json).get(doc)); + int64_t x; + ASSERT_SUCCESS((*doc.begin()).get_int64().get(x)); + ASSERT_EQUAL(x,133); + TEST_SUCCEED(); + } + + bool double_with_trailing() { + TEST_START(); + auto json = R"( 133 badstuff )"_padded; + singlestage::parser parser; + singlestage::document_stream doc; + ASSERT_SUCCESS(parser.iterate_many(json).get(doc)); + double x; + ASSERT_SUCCESS((*doc.begin()).get_double().get(x)); + ASSERT_EQUAL(x,133); + TEST_SUCCEED(); + } + + + bool bool_with_trailing() { + TEST_START(); + auto json = R"( true badstuff )"_padded; + singlestage::parser parser; + singlestage::document_stream doc; + ASSERT_SUCCESS(parser.iterate_many(json).get(doc)); + bool x; + ASSERT_SUCCESS((*doc.begin()).get_bool().get(x)); + ASSERT_EQUAL(x,true); + TEST_SUCCEED(); + } + + bool null_with_trailing() { + TEST_START(); + auto json = R"( null badstuff )"_padded; + singlestage::parser parser; + singlestage::document_stream doc; + ASSERT_SUCCESS(parser.iterate_many(json).get(doc)); + bool n; + ASSERT_SUCCESS((*doc.begin()).is_null().get(n)); + ASSERT_EQUAL(n,true); + TEST_SUCCEED(); + } + + bool run() { + return + issue1977() && + string_with_trailing() && + uint64_with_trailing() && + int64_with_trailing() && + bool_with_trailing() && + null_with_trailing() && + truncated_utf8() && + issue1729() && + fuzzaccess() && + issue1683() && + issue1668() && + issue1668_long() && + simple_document_iteration() && + simple_document_iteration_multiple_batches() && + simple_document_iteration_with_parsing() && + atoms_json() && + doc_index() && + doc_index_multiple_batches() && + source_test() && + truncated() && + truncated_complete_docs() && + truncated_unclosed_string() && + small_window() && + large_window() && + test_leading_spaces() && + test_crazy_leading_spaces() && + adversarial_single_document() && + adversarial_single_document_array() && + document_stream_test() && + document_stream_utf8_test() && + stress_data_race() && + stress_data_race_with_error() && + true; + } +} // document_stream_tests + +int main (int argc, char *argv[]) { + return test_main(argc, argv, document_stream_tests::run); +} \ No newline at end of file diff --git a/tests/singlestage/singlestage_error_location_tests.cpp b/tests/singlestage/singlestage_error_location_tests.cpp new file mode 100644 index 000000000..58032f22b --- /dev/null +++ b/tests/singlestage/singlestage_error_location_tests.cpp @@ -0,0 +1,289 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace error_location_tests { + + bool array() { + TEST_START(); + auto json = R"( [1,2,3] )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + std::vector expected = {'1','2','3'}; + std::vector expected_values = {1,2,3}; + size_t count{0}; + for (auto value : doc) { + int64_t i; + const char* c; + // Must call current_location first because get_int64() will consume values + ASSERT_SUCCESS(doc.current_location().get(c)); + ASSERT_EQUAL(*c, expected[count]); + ASSERT_SUCCESS(value.get_int64().get(i)); + ASSERT_EQUAL(i, expected_values[count]); + count++; + } + ASSERT_EQUAL(count,3); + ASSERT_ERROR(doc.current_location(), OUT_OF_BOUNDS); + TEST_SUCCEED(); + } + + bool object() { + TEST_START(); + auto json = R"( {"a":1, "b":2, "c":3} )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + std::vector expected = {'1','2','3'}; + std::vector expected_values = {1,2,3}; + size_t count{0}; + for (auto field : doc.get_object()) { + int64_t i; + const char* c; + // Must call current_location first because get_int64() will consume values + ASSERT_SUCCESS(doc.current_location().get(c)); + ASSERT_EQUAL(*c,expected[count]); + ASSERT_SUCCESS(field.value().get(i)); + ASSERT_EQUAL(i,expected_values[count]); + count++; + } + ASSERT_EQUAL(count,3); + ASSERT_ERROR(doc.current_location(), OUT_OF_BOUNDS); + TEST_SUCCEED(); + } + + bool json_pointer() { + TEST_START(); + auto json = R"( {"a": [1,2,[3,4,5]], "b": {"c": [1.2, 2.3]}} )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + const char * ptr; + uint64_t i; + ASSERT_SUCCESS(doc.at_pointer("/a/2/1").get(i)); + ASSERT_EQUAL(i, 4); + ASSERT_SUCCESS(doc.current_location().get(ptr)); + std::string expected = ",5]], \"b\": {\"c\": [1.2, 2.3]}} "; + ASSERT_EQUAL(std::string(ptr, expected.size()), expected); + double d; + ASSERT_SUCCESS(doc.at_pointer("/b/c/1").get(d)); + ASSERT_EQUAL(d, 2.3); + ASSERT_SUCCESS(doc.current_location().get(ptr)); + expected = "]}} "; + ASSERT_EQUAL(std::string(ptr, expected.size()), expected); + TEST_SUCCEED(); + } + + bool json_pointer_with_errors() { + TEST_START(); + auto json = R"( {"a": [1,2,[3 4,5]], "b": {"c": [1.2., 2.3]}} )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + const char * ptr; + double d; + ASSERT_ERROR(doc.at_pointer("/b/c/0").get(d), NUMBER_ERROR); + ASSERT_SUCCESS(doc.current_location().get(ptr)); + std::string expected = "1.2., 2.3]}} "; + ASSERT_EQUAL(std::string(ptr, expected.size()), expected); + uint64_t i; + ASSERT_ERROR(doc.at_pointer("/a/2/1").get(i), TAPE_ERROR); + ASSERT_SUCCESS(doc.current_location().get(ptr)); + expected = "4,5]], \"b\": {\"c\": [1.2., 2.3]}} "; + ASSERT_EQUAL(std::string(ptr, expected.size()), expected); + TEST_SUCCEED(); + } + + bool broken_json1() { + TEST_START(); + auto json = " \xc3\x94\xc3\xb8\xe2\x84\xa6{\"a\":1, 3} "_padded; + singlestage::parser parser; + singlestage::document doc; + const char * ptr; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ASSERT_ERROR(doc["a"], INCORRECT_TYPE); + ASSERT_SUCCESS(doc.current_location().get(ptr)); + ASSERT_EQUAL(std::string(ptr, 18), "\xc3\x94\xc3\xb8\xe2\x84\xa6{\"a\":1, 3} "); + TEST_SUCCEED(); + } + + bool broken_json2() { + TEST_START(); + auto json = R"( [[[]] )"_padded; + singlestage::parser parser; + singlestage::document doc; + singlestage::array arr; + const char * ptr; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ASSERT_SUCCESS(doc.get_array().get(arr)); + ASSERT_ERROR(arr.count_elements(), TAPE_ERROR); + ASSERT_SUCCESS(doc.current_location().get(ptr)); + ASSERT_EQUAL(std::string(ptr - 2,2), "] "); + TEST_SUCCEED(); + } + + bool broken_json3() { + TEST_START(); + auto json = R"( [1 1.23, 2] )"_padded; + singlestage::parser parser; + singlestage::document doc; + size_t count{0}; + const char * ptr; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + for (auto val : doc) { + if (count == 1) { + ASSERT_ERROR(val, TAPE_ERROR); + break; + } + count++; + } + ASSERT_EQUAL(count, 1); + ASSERT_SUCCESS(doc.current_location().get(ptr)); + ASSERT_EQUAL(std::string(ptr, strlen("1.23, 2] ")), "1.23, 2] "); + TEST_SUCCEED(); + } + + bool broken_json4() { + TEST_START(); + auto json = R"( {"a":1, 3.5, "b":5} )"_padded; + singlestage::parser parser; + singlestage::document doc; + const char * ptr; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ASSERT_ERROR(doc["b"], TAPE_ERROR); + ASSERT_SUCCESS(doc.current_location().get(ptr)); + ASSERT_EQUAL(std::string(ptr, strlen("3.5, \"b\":5} ")), "3.5, \"b\":5} "); + TEST_SUCCEED(); + } + + bool incomplete_json() { + TEST_START(); + auto json = R"( [1,2,3 )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + const char * ptr; + for (auto val : doc) { + ASSERT_ERROR(val, INCOMPLETE_ARRAY_OR_OBJECT); + } + ASSERT_SUCCESS(doc.current_location().get(ptr)); + ASSERT_EQUAL(std::string(ptr, strlen("[1,2,3 ")), "[1,2,3 "); + TEST_SUCCEED(); + } + + bool boolean_error() { + TEST_START(); + auto json = R"( [tru, fals] )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + for (auto val : doc) { + bool b; + ASSERT_ERROR(val.get(b), INCORRECT_TYPE); + } + ASSERT_ERROR(doc.current_location(), OUT_OF_BOUNDS); + TEST_SUCCEED(); + } + + bool no_such_field() { + TEST_START(); + auto json = R"( {"a":5, "b":4} )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ASSERT_ERROR(doc["c"], NO_SUCH_FIELD); + ASSERT_ERROR(doc.current_location(), OUT_OF_BOUNDS); + TEST_SUCCEED(); + } + + bool object_with_no_such_field() { + TEST_START(); + auto json = R"( {"a":5, "b":4} )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + const char * ptr; + uint64_t i; + ASSERT_SUCCESS(doc["a"].get(i)); + ASSERT_EQUAL(i, 5); + ASSERT_SUCCESS(doc.current_location().get(ptr)); + ASSERT_EQUAL(ptr, ", \"b\":4} "); + ASSERT_ERROR(doc["c"], NO_SUCH_FIELD); + ASSERT_SUCCESS(doc.current_location().get(ptr)); + ASSERT_EQUAL(ptr, ", \"b\":4} "); + TEST_SUCCEED(); + } + + bool number_parsing_error() { + TEST_START(); + auto json = R"( [13.34.514] )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + const char * ptr; + double d; + ASSERT_ERROR(doc.at_pointer("/0").get_double().get(d), NUMBER_ERROR); + ASSERT_SUCCESS(doc.current_location().get(ptr)); + ASSERT_EQUAL(ptr, "13.34.514] "); + TEST_SUCCEED(); + } + bool number_parsing_root_error() { + TEST_START(); + auto json = R"( 13.34.514 )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + const char * ptr; + double d; + ASSERT_ERROR(doc.get_double().get(d), NUMBER_ERROR); + ASSERT_SUCCESS(doc.current_location().get(ptr)); + ASSERT_EQUAL(ptr, "13.34.514 "); + TEST_SUCCEED(); + } + bool current_location_in_value() { + TEST_START(); + auto json = R"( {"a": {"b": "c"}} )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + simdjson_result a = doc["a"]; + ASSERT_SUCCESS(a); + const char * ptr; + ASSERT_SUCCESS(a.current_location().get(ptr)); + ASSERT_EQUAL(ptr, R"({"b": "c"}} )"); + simdjson_result b = a["b"]; + ASSERT_SUCCESS(b); + ASSERT_SUCCESS(b.current_location().get(ptr)); + ASSERT_EQUAL(ptr, R"("c"}} )"); + double d; + ASSERT_ERROR(b.get_double().get(d), INCORRECT_TYPE); + ASSERT_SUCCESS(b.current_location().get(ptr)); + ASSERT_EQUAL(ptr, R"("c"}} )"); + TEST_SUCCEED(); + } + + bool run() { + return array() && + object() && + json_pointer() && + json_pointer_with_errors() && + broken_json1() && + broken_json2() && + broken_json3() && + broken_json4() && + incomplete_json() && + boolean_error() && + no_such_field() && + object_with_no_such_field() && + number_parsing_error() && + number_parsing_root_error() && + current_location_in_value() && + true; + } + +} // namespace error_location_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, error_location_tests::run); +} \ No newline at end of file diff --git a/tests/singlestage/singlestage_error_tests.cpp b/tests/singlestage/singlestage_error_tests.cpp new file mode 100644 index 000000000..1ce75c15d --- /dev/null +++ b/tests/singlestage/singlestage_error_tests.cpp @@ -0,0 +1,393 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace error_tests { + using namespace std; + + bool badbadjson() { + TEST_START(); + singlestage::parser parser; + auto json_string = R"({ + "main": "therain"_"in_spain"_ +})"_padded; + singlestage::document document; + auto error = parser.iterate(json_string).get(document); + if(error != simdjson::SUCCESS) { + return false; + } + singlestage::object obj; + error = document.get_object().get(obj); + if(error != simdjson::SUCCESS) { + return false; + } + std::string_view name_value{}; + error = obj["name"].get_string().get(name_value); + ASSERT_ERROR(error,TAPE_ERROR); + // Check for "main" field + std::string_view main_value{}; + error = obj["main"].get_string().get(main_value); + ASSERT_ERROR(error,TAPE_ERROR); + TEST_SUCCEED(); + } + + + bool badbadjson2() { + TEST_START(); + singlestage::parser parser; + auto json_string = R"({ + "main": "therain"_"in_spain"_ +})"_padded; + singlestage::document document; + auto error = parser.iterate(json_string).get(document); + if(error != simdjson::SUCCESS) { + return false; + } + singlestage::object obj; + error = document.get_object().get(obj); + if(error != simdjson::SUCCESS) { + return false; + } + + std::string_view main_value{}; + error = obj["main"].get_string().get(main_value); + ASSERT_ERROR(error, simdjson::SUCCESS); + std::string_view name_value{}; + error = obj["name"].get_string().get(name_value); + ASSERT_ERROR(error,TAPE_ERROR); + TEST_SUCCEED(); + } + + bool issue1834() { + TEST_START(); + singlestage::parser parser; + auto json = "[[]"_padded; + json.data()[json.size()] = ']'; + auto doc = parser.iterate(json); + size_t cnt{}; + auto error = doc.count_elements().get(cnt); + return error != simdjson::SUCCESS; + } + bool issue1834_2() { + TEST_START(); + singlestage::parser parser; + auto json = "{\"a\":{}"_padded; + json.data()[json.size()] = '}'; + auto doc = parser.iterate(json); + size_t cnt{}; + auto error = doc.count_fields().get(cnt); + return error != simdjson::SUCCESS; + } + bool empty_document_error() { + TEST_START(); + singlestage::parser parser; + auto json = ""_padded; + ASSERT_ERROR( parser.iterate(json), EMPTY ); + TEST_SUCCEED(); + } + bool raw_json_string_error() { + TEST_START(); + singlestage::parser parser; + auto json = "{\"haha\":{\"df2\":3.5, \"df3\": \"fd\"}}"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(json).get(doc) ); + singlestage::raw_json_string rawjson; + ASSERT_ERROR( doc.get_raw_json_string().get(rawjson), INCORRECT_TYPE ); + TEST_SUCCEED(); + } +#if SIMDJSON_EXCEPTIONS + bool raw_json_string_except() { + TEST_START(); + singlestage::parser parser; + auto json = "{\"haha\":{\"df2\":3.5, \"df3\": \"fd\"}}"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(json).get(doc) ); + try { + singlestage::raw_json_string rawjson = doc.get_raw_json_string(); + (void)rawjson; + TEST_FAIL("Should have thrown an exception!") + } catch(simdjson_error& e) { + ASSERT_ERROR(e.error(), INCORRECT_TYPE); + TEST_SUCCEED(); + } + } + bool raw_json_string_except_with_io() { + TEST_START(); + singlestage::parser parser; + auto json = "{\"haha\":{\"df2\":3.5, \"df3\": \"fd\"}}"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(json).get(doc) ); + try { + auto rawjson = doc.get_raw_json_string(); + std::cout << rawjson; + TEST_FAIL("Should have thrown an exception!") + } catch(simdjson_error& e) { + ASSERT_ERROR(e.error(), INCORRECT_TYPE); + TEST_SUCCEED(); + } + } + +#endif + bool parser_max_capacity() { + TEST_START(); + singlestage::parser parser(1); // max_capacity set to 1 byte + padded_string json; + ASSERT_SUCCESS( padded_string::load(TWITTER_JSON).get(json) ); + auto error = parser.iterate(json); + ASSERT_ERROR(error,CAPACITY); + TEST_SUCCEED(); + } + + bool simple_error_example() { + TEST_START(); + singlestage::parser parser; + auto json = R"({"bad number":3.14.1 })"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(json).get(doc) ); + double x; + ASSERT_ERROR( doc["bad number"].get_double().get(x), NUMBER_ERROR ); + TEST_SUCCEED(); + } + +#if SIMDJSON_EXCEPTIONS + bool simple_error_example_except() { + TEST_START(); + singlestage::parser parser; + auto json = R"({"bad number":3.14.1 })"_padded; + try { + singlestage::document doc = parser.iterate(json); + double x = doc["bad number"].get_double(); + TEST_FAIL("should throw got: "+std::to_string(x)) + } catch(simdjson_error& e) { + ASSERT_ERROR(e.error(), NUMBER_ERROR); + } + TEST_SUCCEED(); + } +#endif + + bool get_fail_then_succeed_bool() { + TEST_START(); + auto json = R"({ "val" : true })"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + simdjson_result val = doc["val"]; + // Get everything that can fail in both forward and backwards order + bool is_null_value; + ASSERT_SUCCESS( val.is_null().get(is_null_value) ); + ASSERT_EQUAL( is_null_value, false ); + ASSERT_ERROR( val.get_array(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_string(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_double(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_uint64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_int64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_object(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_int64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_uint64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_double(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_string(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_array(), INCORRECT_TYPE ); + ASSERT_SUCCESS( val.get_bool() ); + ASSERT_SUCCESS( val.is_null().get(is_null_value) ); + ASSERT_EQUAL( is_null_value, false ); + TEST_SUCCEED(); + })); + SUBTEST("singlestage::value", test_singlestage_doc(json, [&](auto doc) { + singlestage::value val; + ASSERT_SUCCESS( doc["val"].get(val) ); + // Get everything that can fail in both forward and backwards order + bool is_null_value; + ASSERT_SUCCESS( val.is_null().get(is_null_value) ); + ASSERT_EQUAL( is_null_value, false ); + ASSERT_ERROR( val.get_array(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_string(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_double(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_uint64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_int64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_object(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_int64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_uint64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_double(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_string(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_array(), INCORRECT_TYPE ); + ASSERT_SUCCESS( val.get_bool() ); + ASSERT_SUCCESS( val.is_null().get(is_null_value) ); + ASSERT_EQUAL( is_null_value, false ); + TEST_SUCCEED(); + })); + json = R"(true)"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](simdjson_result val) { + // Get everything that can fail in both forward and backwards order + bool is_null_value; + ASSERT_SUCCESS( val.is_null().get(is_null_value) ); + ASSERT_EQUAL( is_null_value, false ); + ASSERT_ERROR( val.get_array(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_string(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_double(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_uint64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_int64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_object(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_int64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_uint64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_double(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_string(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_array(), INCORRECT_TYPE ); + ASSERT_SUCCESS( val.get_bool()); + ASSERT_SUCCESS( val.is_null().get(is_null_value) ); + ASSERT_EQUAL( is_null_value, false ); + TEST_SUCCEED(); + })); + SUBTEST("singlestage::document", test_singlestage_doc(json, [&](auto doc) { + singlestage::document val; + ASSERT_SUCCESS( std::move(doc).get(val) ); // Get everything that can fail in both forward and backwards order + bool is_null_value; + ASSERT_SUCCESS( val.is_null().get(is_null_value) ); + ASSERT_EQUAL( is_null_value, false ); + ASSERT_ERROR( val.get_array(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_string(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_double(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_uint64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_int64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_object(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_int64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_uint64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_double(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_string(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_array(), INCORRECT_TYPE ); + ASSERT_SUCCESS( val.get_bool() ); + ASSERT_SUCCESS( val.is_null().get(is_null_value) ); + ASSERT_EQUAL( is_null_value, false ); + TEST_SUCCEED(); + })); + + TEST_SUCCEED(); + } + + bool get_fail_then_succeed_null() { + TEST_START(); + auto json = R"({ "val" : null })"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + simdjson_result val = doc["val"]; + // Get everything that can fail in both forward and backwards order + ASSERT_ERROR( val.get_bool(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_array(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_string(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_double(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_uint64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_int64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_object(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_int64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_uint64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_double(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_string(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_array(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_bool(), INCORRECT_TYPE ); + bool is_null_value; + ASSERT_SUCCESS( val.is_null().get(is_null_value) ); + ASSERT_EQUAL( is_null_value, true ); + TEST_SUCCEED(); + })); + SUBTEST("singlestage::value", test_singlestage_doc(json, [&](auto doc) { + singlestage::value val; + ASSERT_SUCCESS( doc["val"].get(val) ); + // Get everything that can fail in both forward and backwards order + + ASSERT_ERROR( val.get_bool(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_array(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_string(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_double(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_uint64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_int64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_object(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_int64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_uint64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_double(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_string(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_array(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_bool(), INCORRECT_TYPE ); + bool is_null_value; + ASSERT_SUCCESS( val.is_null().get(is_null_value) ); + ASSERT_EQUAL( is_null_value, true ); + TEST_SUCCEED(); + })); + json = R"(null)"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](simdjson_result val) { + // Get everything that can fail in both forward and backwards order + ASSERT_ERROR( val.get_bool(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_array(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_string(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_double(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_uint64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_int64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_object(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_int64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_uint64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_double(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_string(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_array(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_bool(), INCORRECT_TYPE ); + bool is_null_value; + ASSERT_SUCCESS( val.is_null().get(is_null_value) ); + ASSERT_EQUAL( is_null_value, true ); + TEST_SUCCEED(); + })); + SUBTEST("singlestage::document", test_singlestage_doc(json, [&](auto doc) { + singlestage::document val; + ASSERT_SUCCESS( std::move(doc).get(val) ); // Get everything that can fail in both forward and backwards order + ASSERT_ERROR( val.get_bool(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_array(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_string(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_double(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_uint64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_int64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_object(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_int64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_uint64(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_double(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_string(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_array(), INCORRECT_TYPE ); + ASSERT_ERROR( val.get_bool(), INCORRECT_TYPE ); + bool is_null_value; + ASSERT_SUCCESS( val.is_null().get(is_null_value) ); + ASSERT_EQUAL( is_null_value, true ); + TEST_SUCCEED(); + })); + TEST_SUCCEED(); + } + + bool invalid_type() { + TEST_START(); + SINGLESTAGE_SUBTEST("]", "]", doc.type().error() == TAPE_ERROR); + SINGLESTAGE_SUBTEST("}", "}", doc.type().error() == TAPE_ERROR); + SINGLESTAGE_SUBTEST(":", ":", doc.type().error() == TAPE_ERROR); + SINGLESTAGE_SUBTEST(",", ",", doc.type().error() == TAPE_ERROR); + SINGLESTAGE_SUBTEST("+", "+", doc.type().error() == TAPE_ERROR); + TEST_SUCCEED(); + } + + bool run() { + return + badbadjson() && + badbadjson2() && + issue1834() && + issue1834_2() && +#if SIMDJSON_EXCEPTIONS + raw_json_string_except() && + raw_json_string_except_with_io() && +#endif + raw_json_string_error() && + empty_document_error() && + parser_max_capacity() && + get_fail_then_succeed_bool() && + get_fail_then_succeed_null() && + invalid_type() && + simple_error_example() && +#if SIMDJSON_EXCEPTIONS + simple_error_example_except() && +#endif + true; + } +} + +int main(int argc, char *argv[]) { + return test_main(argc, argv, error_tests::run); +} diff --git a/tests/singlestage/singlestage_iterate_many_csv.cpp b/tests/singlestage/singlestage_iterate_many_csv.cpp new file mode 100644 index 000000000..5b20e5371 --- /dev/null +++ b/tests/singlestage/singlestage_iterate_many_csv.cpp @@ -0,0 +1,160 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +#include + +using namespace simdjson; + +namespace iterate_many_csv_tests { +using namespace std; + +bool normal() { + TEST_START(); + auto json = R"( 1, 2, 3, 4, "a", "b", "c", {"hello": "world"} , [1, 2, 3])"_padded; + singlestage::parser parser; + singlestage::document_stream doc_stream; + ASSERT_SUCCESS(parser.iterate_many(json, json.size(), true).get(doc_stream)); + + for (auto doc : doc_stream) + { + ASSERT_SUCCESS(doc); + } + + TEST_SUCCEED(); +} + +bool small_batch_size() { + TEST_START(); + auto json = R"( 1, 2, 3, 4, "a", "b", "c", {"hello": "world"} , [1, 2, 3])"_padded; + singlestage::parser parser; + singlestage::document_stream doc_stream; + ASSERT_SUCCESS(parser.iterate_many(json, 32, true).get(doc_stream)); + + for (auto doc : doc_stream) + { + ASSERT_SUCCESS(doc); + } + + TEST_SUCCEED(); +} + +bool trailing_comma() { + TEST_START(); + auto json = R"(1,)"_padded; + singlestage::parser parser; + singlestage::document_stream doc_stream; + ASSERT_SUCCESS(parser.iterate_many(json, json.size(), true).get(doc_stream)); + + for (auto doc : doc_stream) + { + ASSERT_SUCCESS(doc); + } + + TEST_SUCCEED(); +} + +bool check_parsed_values() { + TEST_START(); + + auto json = R"( 1 , "a" , [100, 1] , {"hello" : "world"} , )"_padded; + singlestage::parser parser; + singlestage::document_stream doc_stream; + ASSERT_SUCCESS(parser.iterate_many(json, json.size(), true).get(doc_stream)); + + auto begin = doc_stream.begin(); + auto end = doc_stream.end(); + int cnt = 0; + auto it = begin; + for (; it != end && cnt < 4; ++it, ++cnt) { + auto doc = *it; + switch (cnt) + { + case 0: + { + int64_t actual; + ASSERT_SUCCESS(doc.get_int64().get(actual)); + ASSERT_EQUAL(actual, 1); + break; + } + case 1: + { + std::string_view sv; + ASSERT_SUCCESS(doc.get_string().get(sv)); + ASSERT_EQUAL(sv, "a"); + break; + } + case 2: + { + std::vector expected{100, 1}; + singlestage::array arr; + ASSERT_SUCCESS(doc.get_array().get(arr)); + size_t element_count; + ASSERT_SUCCESS(arr.count_elements().get(element_count)); + ASSERT_EQUAL(element_count, 2); + int i = 0; + for (auto a : arr) + { + int64_t actual; + ASSERT_SUCCESS(a.get(actual)); + ASSERT_EQUAL(actual, expected[i++]); + } + break; + } + case 3: + { + singlestage::object obj; + ASSERT_SUCCESS(doc.get_object().get(obj)); + std::string_view sv; + obj.find_field("hello").get(sv); + ASSERT_EQUAL(sv, "world"); + break; + } + default: + TEST_FAIL("Too many cases") + } + } + + ASSERT_EQUAL(cnt, 4); + ASSERT_TRUE(!(it != end)); + + TEST_SUCCEED(); +} + +#if SIMDJSON_EXCEPTIONS + +bool leading_comma() { + TEST_START(); + auto json = R"(,1)"_padded; + singlestage::parser parser; + singlestage::document_stream doc_stream; + ASSERT_SUCCESS(parser.iterate_many(json, json.size(), true).get(doc_stream)); + + try { + auto begin = doc_stream.begin(); + auto end = doc_stream.end(); + for (auto it = begin; it != end; ++it) {} + } catch (simdjson_error& e) { + ASSERT_ERROR(e.error(), TAPE_ERROR); + } + + TEST_SUCCEED(); +} + +#endif + +bool run() { + return normal() && + small_batch_size() && + trailing_comma() && + check_parsed_values() && +#if SIMDJSON_EXCEPTIONS + leading_comma() && +#endif + true; +} + +} + +int main(int argc, char *argv[]) { + return test_main(argc, argv, iterate_many_csv_tests::run); +} diff --git a/tests/singlestage/singlestage_json_pointer_tests.cpp b/tests/singlestage/singlestage_json_pointer_tests.cpp new file mode 100644 index 000000000..53a49f898 --- /dev/null +++ b/tests/singlestage/singlestage_json_pointer_tests.cpp @@ -0,0 +1,491 @@ +#include "simdjson.h" +#include "test_singlestage.h" +#include + +using namespace simdjson; + +namespace json_pointer_tests { + const padded_string TEST_JSON = R"( + { + "/~01abc": [ + 0, + { + "\\\" 0": [ + "value0", + "value1" + ] + } + ], + "0": "0 ok", + "01": "01 ok", + "": "empty ok", + "arr": [] + } + )"_padded; + + const padded_string TEST_RFC_JSON = R"( + { + "foo": ["bar", "baz"], + "": 0, + "a/b": 1, + "c%d": 2, + "e^f": 3, + "g|h": 4, + "i\\j": 5, + "k\"l": 6, + " ": 7, + "m~n": 8 + } + )"_padded; + + bool run_success_test(const padded_string & json,std::string_view json_pointer,std::string expected) { + TEST_START(); + std::cout <<":"<< json_pointer< measured; + ASSERT_SUCCESS(parser.iterate(cars_json).get(cars)); + for (auto car_element : cars) { + double x; + ASSERT_SUCCESS(car_element.at_pointer("/tire_pressure/1").get(x)); + measured.push_back(x); + } + + std::vector expected = {39.9, 31, 30}; + if (measured != expected) { return false; } + TEST_SUCCEED(); + } + + bool document_as_scalar() { + TEST_START(); + auto number_json = R"( 1 )"_padded; + auto null_json = R"( null )"_padded; + auto string_json = R"( "a" )"_padded; + auto true_json = R"( true )"_padded; + auto false_json = R"( false )"_padded; + auto object_json = R"( {} )"_padded; + auto array_json = R"( {} )"_padded; + singlestage::parser parser; + singlestage::document doc; + singlestage::value val; + bool is_scalar; + std::cout << " checking number"<< std::endl; + ASSERT_SUCCESS(parser.iterate(number_json).get(doc)); + ASSERT_SUCCESS(doc.is_scalar().get(is_scalar)); + ASSERT_TRUE(is_scalar); + ASSERT_ERROR(doc.at_pointer("").get(val), simdjson::SCALAR_DOCUMENT_AS_VALUE); + std::cout << " checking null"<< std::endl; + ASSERT_SUCCESS(parser.iterate(null_json).get(doc)); + ASSERT_SUCCESS(doc.is_scalar().get(is_scalar)); + ASSERT_TRUE(is_scalar); + ASSERT_ERROR(doc.at_pointer("").get(val), simdjson::SCALAR_DOCUMENT_AS_VALUE); + std::cout << " checking string"<< std::endl; + ASSERT_SUCCESS(parser.iterate(string_json).get(doc)); + ASSERT_SUCCESS(doc.is_scalar().get(is_scalar)); + ASSERT_TRUE(is_scalar); + ASSERT_ERROR(doc.at_pointer("").get(val), simdjson::SCALAR_DOCUMENT_AS_VALUE); + std::cout << " checking false"<< std::endl; + ASSERT_SUCCESS(parser.iterate(false_json).get(doc)); + ASSERT_SUCCESS(doc.is_scalar().get(is_scalar)); + ASSERT_TRUE(is_scalar); + ASSERT_ERROR(doc.at_pointer("").get(val), simdjson::SCALAR_DOCUMENT_AS_VALUE); + std::cout << " checking true"<< std::endl; + ASSERT_SUCCESS(parser.iterate(true_json).get(doc)); + ASSERT_SUCCESS(doc.is_scalar().get(is_scalar)); + ASSERT_TRUE(is_scalar); + ASSERT_ERROR(doc.at_pointer("").get(val), simdjson::SCALAR_DOCUMENT_AS_VALUE); + std::cout << " checking object"<< std::endl; + ASSERT_SUCCESS(parser.iterate(object_json).get(doc)); + ASSERT_SUCCESS(doc.is_scalar().get(is_scalar)); + ASSERT_FALSE(is_scalar); + ASSERT_SUCCESS(doc.at_pointer("").get(val)); + std::cout << " checking array"<< std::endl; + ASSERT_SUCCESS(parser.iterate(array_json).get(doc)); + ASSERT_SUCCESS(doc.is_scalar().get(is_scalar)); + ASSERT_FALSE(is_scalar); + ASSERT_SUCCESS(doc.at_pointer("").get(val)); + TEST_SUCCEED(); + } + + + bool many_json_pointers() { + TEST_START(); + auto cars_json = R"( [ + { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, + { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, + { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] } + ] )"_padded; + + singlestage::parser parser; + singlestage::document cars; + std::vector measured; + ASSERT_SUCCESS(parser.iterate(cars_json).get(cars)); + for (int i = 0; i < 3; i++) { + double x; + std::string json_pointer = std::string("/") + std::to_string(i) + std::string("/tire_pressure/1"); + ASSERT_SUCCESS(cars.at_pointer(json_pointer).get(x)); + measured.push_back(x); + } + + std::vector expected = {39.9, 31, 30}; + if (measured != expected) { return false; } + TEST_SUCCEED(); + } + bool run_broken_tests() { + TEST_START(); + singlestage::parser parser; + singlestage::document doc; + singlestage::value v; + std::string_view val; + + auto invalid_escape_key = R"( {"hello": [0,1,2,3], "te\est": "foo", "bool": true, "num":1234, "success":"yes"} )"_padded; + auto invalid_escape_value = R"( {"hello": [0,1,2,3], "test": "fo\eo", "bool": true, "num":1234, "success":"yes"} )"_padded; + auto invalid_escape_value_at_jp = R"( {"hello": [0,1,2,3], "test": "foo", "bool": true, "num":1234, "success":"y\es"} )"_padded; + auto unclosed_object = R"( {"test": "foo", "bool": true, "num":1234, "success":"yes" )"_padded; + auto missing_bracket_before = R"( {"hello": [0,1,2,3, "test": "foo", "bool": true, "num":1234, "success":"yes"} )"_padded; + auto missing_bracket_after = R"( {"test": "foo", "bool": true, "num":1234, "success":"yes", "hello":[0,1,2,3} )"_padded; + + std::string json_pointer = "/success"; + std::cout << "\t- invalid_escape_key" << std::endl; + ASSERT_SUCCESS(parser.iterate(invalid_escape_key).get(doc)); + ASSERT_SUCCESS(doc.at_pointer(json_pointer).get(val)); + std::cout << "\t- invalid_escape_value" << std::endl; + ASSERT_SUCCESS(parser.iterate(invalid_escape_value).get(doc)); + ASSERT_SUCCESS(doc.at_pointer(json_pointer).get(val)); + std::cout << "\t- invalid_escape_value_at_jp_nomat" << std::endl; + ASSERT_SUCCESS(parser.iterate(invalid_escape_value_at_jp).get(doc)); + ASSERT_SUCCESS(doc.at_pointer(json_pointer).get(v)); + std::cout << "\t- invalid_escape_value_at_jp" << std::endl; + ASSERT_SUCCESS(parser.iterate(invalid_escape_value_at_jp).get(doc)); + ASSERT_ERROR(doc.at_pointer(json_pointer).get(val), simdjson::STRING_ERROR); + std::cout << "\t- unclosed_object" << std::endl; + ASSERT_SUCCESS(parser.iterate(unclosed_object).get(doc)); + ASSERT_ERROR(doc.at_pointer(json_pointer).get(val), simdjson::INCOMPLETE_ARRAY_OR_OBJECT); + std::cout << "\t- missing_bracket_before" << std::endl; + ASSERT_SUCCESS(parser.iterate(missing_bracket_before).get(doc)); + ASSERT_ERROR(doc.at_pointer(json_pointer).get(val), simdjson::TAPE_ERROR); + std::cout << "\t- missing_bracket_after" << std::endl; + ASSERT_SUCCESS(parser.iterate(missing_bracket_after).get(doc)); + ASSERT_SUCCESS(doc.at_pointer(json_pointer).get(val)); + TEST_SUCCEED(); + } + + + bool many_json_pointers_object_array() { + TEST_START(); + auto dogcatpotato = R"( { "dog" : [1,2,3], "cat" : [5, 6, 7], "potato" : [1234]})"_padded; + + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(dogcatpotato).get(doc)); + singlestage::object obj; + ASSERT_SUCCESS(doc.get_object().get(obj)); + int64_t x; + ASSERT_SUCCESS(obj.at_pointer("/dog/1").get(x)); + ASSERT_EQUAL(x, 2); + ASSERT_SUCCESS(obj.at_pointer("/potato/0").get(x)); + ASSERT_EQUAL(x, 1234); + TEST_SUCCEED(); + } + bool many_json_pointers_object() { + TEST_START(); + auto cfoofoo2 = R"( { "c" :{ "foo": { "a": [ 10, 20, 30 ] }}, "d": { "foo2": { "a": [ 10, 20, 30 ] }} , "e": 120 })"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(cfoofoo2).get(doc)); + singlestage::object obj; + ASSERT_SUCCESS(doc.get_object().get(obj)); + int64_t x; + ASSERT_SUCCESS(obj.at_pointer("/c/foo/a/1").get(x)); + ASSERT_EQUAL(x, 20); + ASSERT_SUCCESS(obj.at_pointer("/d/foo2/a/2").get(x)); + ASSERT_EQUAL(x, 30); + ASSERT_SUCCESS(obj.at_pointer("/e").get(x)); + ASSERT_EQUAL(x, 120); + TEST_SUCCEED(); + } + bool many_json_pointers_array() { + TEST_START(); + auto cfoofoo2 = R"( [ 111, 2, 3, { "foo": { "a": [ 10, 20, 33 ] }}, { "foo2": { "a": [ 10, 20, 30 ] }}, 1001 ])"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(cfoofoo2).get(doc)); + singlestage::array arr; + ASSERT_SUCCESS(doc.get_array().get(arr)); + int64_t x; + ASSERT_SUCCESS(arr.at_pointer("/3/foo/a/1").get(x)); + ASSERT_EQUAL(x, 20); + TEST_SUCCEED(); + } + struct car_type { + std::string make; + std::string model; + uint64_t year; + std::vector tire_pressure; + car_type(std::string_view _make, std::string_view _model, uint64_t _year, + std::vector&& _tire_pressure) : + make{_make}, model{_model}, year(_year), tire_pressure(_tire_pressure) {} + }; + + bool json_pointer_invalidation() { + TEST_START(); + auto cars_json = R"( [ + { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, + { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, + { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] } + ] )"_padded; + + singlestage::parser parser; + singlestage::document cars; + std::vector measured; + ASSERT_SUCCESS(parser.iterate(cars_json).get(cars)); + std::vector content; + for (int i = 0; i < 3; i++) { + singlestage::object obj; + std::string json_pointer = std::string("/") + std::to_string(i); + // Each successive at_pointer call invalidates + // previously parsed values, strings, objects and array. + ASSERT_SUCCESS(cars.at_pointer(json_pointer).get(obj)); + // We materialize the object. + std::string_view make; + ASSERT_SUCCESS(obj["make"].get(make)); + std::string_view model; + ASSERT_SUCCESS(obj["model"].get(model)); + uint64_t year; + ASSERT_SUCCESS(obj["year"].get(year)); + // We materialize the array. + singlestage::array arr; + ASSERT_SUCCESS(obj["tire_pressure"].get(arr)); + std::vector values; + for(auto x : arr) { + double value_double; + ASSERT_SUCCESS(x.get(value_double)); + values.push_back(value_double); + } + content.emplace_back(make, model, year, std::move(values)); + } + std::string expected[] = {"Toyota", "Kia", "Toyota"}; + int i = 0; + for (car_type c : content) { + std::cout << c.make << " " << c.model << " " << c.year << "\n"; + ASSERT_EQUAL(expected[i++], c.make); + } + TEST_SUCCEED(); + } + +#if SIMDJSON_EXCEPTIONS + bool json_pointer_invalidation_exceptions() { + TEST_START(); + auto cars_json = R"( [ + { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, + { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, + { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] } + ] )"_padded; + + singlestage::parser parser; + std::vector measured; + singlestage::document cars = parser.iterate(cars_json); + std::vector content; + for (int i = 0; i < 3; i++) { + std::string json_pointer = std::string("/") + std::to_string(i); + // Each successive at_pointer call invalidates + // previously parsed values, strings, objects and array. + singlestage::object obj(cars.at_pointer(json_pointer).get_object()); + // We materialize the object. + std::string_view make = obj["make"]; + std::string_view model = obj["model"]; + uint64_t year(obj["year"]); + // We materialize the array. + singlestage::array arr(obj["tire_pressure"].get_array()); + std::vector values; + for(auto x : arr) { + double value_double(x.get_double()); + values.push_back(value_double); + } + content.emplace_back(make, model, year, std::move(values)); + } + std::string expected[] = {"Toyota", "Kia", "Toyota"}; + for (car_type c : content) { + std::cout << c.make << " " << c.model << " " << c.year << "\n"; + } + TEST_SUCCEED(); + } +#endif + bool run() { + return +#if SIMDJSON_EXCEPTIONS + json_pointer_invalidation_exceptions() && +#endif + many_json_pointers_array() && + many_json_pointers_object() && + many_json_pointers_object_array() && + run_broken_tests() && + json_pointer_invalidation() && + demo_test() && + demo_relative_path() && + run_success_test(TEST_RFC_JSON,"/foo",R"(["bar", "baz"])") && + run_success_test(TEST_RFC_JSON,"/foo/0",R"("bar")") && + run_success_test(TEST_RFC_JSON,"/",R"(0)") && + run_success_test(TEST_RFC_JSON,"/a~1b",R"(1)") && + run_success_test(TEST_RFC_JSON,"/c%d",R"(2)") && + run_success_test(TEST_RFC_JSON,"/e^f",R"(3)") && + run_success_test(TEST_RFC_JSON,"/g|h",R"(4)") && + run_success_test(TEST_RFC_JSON,R"(/i\\j)",R"(5)") && + run_success_test(TEST_RFC_JSON,R"(/k\"l)",R"(6)") && + run_success_test(TEST_RFC_JSON,"/ ",R"(7)") && + run_success_test(TEST_RFC_JSON,"/m~0n",R"(8)") && + run_success_test(TEST_RFC_JSON,"",R"({ + "foo": ["bar", "baz"], + "": 0, + "a/b": 1, + "c%d": 2, + "e^f": 3, + "g|h": 4, + "i\\j": 5, + "k\"l": 6, + " ": 7, + "m~n": 8 + })") && + run_success_test(TEST_JSON, "", R"({ + "/~01abc": [ + 0, + { + "\\\" 0": [ + "value0", + "value1" + ] + } + ], + "0": "0 ok", + "01": "01 ok", + "": "empty ok", + "arr": [] + })") && + run_success_test(TEST_JSON, R"(/~1~001abc)", R"([ + 0, + { + "\\\" 0": [ + "value0", + "value1" + ] + } + ])") && + run_success_test(TEST_JSON, R"(/~1~001abc/1)", R"({ + "\\\" 0": [ + "value0", + "value1" + ] + })") && + run_success_test(TEST_JSON, R"(/~1~001abc/1/\\\" 0)", R"([ + "value0", + "value1" + ])") && + run_success_test(TEST_JSON, R"(/~1~001abc/1/\\\" 0/0)", "\"value0\"") && + run_success_test(TEST_JSON, R"(/~1~001abc/1/\\\" 0/1)", "\"value1\"") && + run_success_test(TEST_JSON, "/arr", R"([])") && + run_success_test(TEST_JSON, "/0", "\"0 ok\"") && + run_success_test(TEST_JSON, "/01", "\"01 ok\"") && + run_success_test(TEST_JSON, "", R"({ + "/~01abc": [ + 0, + { + "\\\" 0": [ + "value0", + "value1" + ] + } + ], + "0": "0 ok", + "01": "01 ok", + "": "empty ok", + "arr": [] + })") && + run_failure_test(TEST_JSON, R"(/~1~001abc/1/\\\" 0/2)", INDEX_OUT_OF_BOUNDS) && + run_failure_test(TEST_JSON, "/arr/0", INDEX_OUT_OF_BOUNDS) && + run_failure_test(TEST_JSON, "~1~001abc", INVALID_JSON_POINTER) && + run_failure_test(TEST_JSON, "/~01abc", NO_SUCH_FIELD) && + run_failure_test(TEST_JSON, "/~1~001abc/01", INVALID_JSON_POINTER) && + run_failure_test(TEST_JSON, "/~1~001abc/", INVALID_JSON_POINTER) && + run_failure_test(TEST_JSON, "/~1~001abc/-", INDEX_OUT_OF_BOUNDS) && + many_json_pointers() && + document_as_scalar() && + true; + } +} // json_pointer_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, json_pointer_tests::run); +} \ No newline at end of file diff --git a/tests/singlestage/singlestage_key_string_tests.cpp b/tests/singlestage/singlestage_key_string_tests.cpp new file mode 100644 index 000000000..dc8e3be6d --- /dev/null +++ b/tests/singlestage/singlestage_key_string_tests.cpp @@ -0,0 +1,33 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace key_string_tests { +#if SIMDJSON_EXCEPTIONS + bool parser_key_value() { + TEST_START(); + singlestage::parser parser; + const padded_string json = R"({ "1": "1", "2": "2", "3": "3", "abc": "abc", "\u0075": "\u0075" })"_padded; + auto doc = parser.iterate(json); + for(auto field : doc.get_object()) { + std::string_view keyv = field.unescaped_key(); + std::string_view valuev = field.value(); + if(keyv != valuev) { return false; } + } + return true; + } +#endif // SIMDJSON_EXCEPTIONS + bool run() { + return +#if SIMDJSON_EXCEPTIONS + parser_key_value() && +#endif // SIMDJSON_EXCEPTIONS + true; + } + +} // namespace key_string_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, key_string_tests::run); +} diff --git a/tests/singlestage/singlestage_log_error_tests.cpp b/tests/singlestage/singlestage_log_error_tests.cpp new file mode 100644 index 000000000..16ab409c2 --- /dev/null +++ b/tests/singlestage/singlestage_log_error_tests.cpp @@ -0,0 +1,67 @@ +#define SIMDJSON_VERBOSE_LOGGING 1 +#include "simdjson.h" +#include "test_singlestage.h" + +#include +#include +#include + +using namespace simdjson; + +namespace log_error_tests { +#if SIMDJSON_EXCEPTIONS + +using namespace std; + +bool tape_error() +{ + TEST_START(); + auto json = R"( {"a", "hello"} )"_padded; + singlestage::parser parser; + try { + singlestage::document doc = parser.iterate(json); + std::cout << doc["a"] << std::endl; + TEST_FAIL("Should have thrown an exception!") + } catch (simdjson_error& e) { + ASSERT_ERROR(e.error(), TAPE_ERROR); + } + TEST_SUCCEED(); +} + +bool no_such_field() +{ + TEST_START(); + auto json = R"( {"a": "hello"} )"_padded; + singlestage::parser parser; + try { + singlestage::document doc = parser.iterate(json); + std::cout << doc["missing_key"] << std::endl; + TEST_FAIL("Should have thrown an exception!") + } catch (simdjson_error& e) { + ASSERT_ERROR(e.error(), NO_SUCH_FIELD); + } + TEST_SUCCEED(); +} +#endif // SIMDJSON_EXCEPTIONS + +bool run() +{ + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe + std::string str = "SIMDJSON_LOG_LEVEL=ERROR"; + putenv(str.data()); + bool rc = +#if SIMDJSON_EXCEPTIONS + tape_error() && + no_such_field() && +#endif // #if SIMDJSON_EXCEPTIONS + true; + SIMDJSON_POP_DISABLE_WARNINGS + return rc; +} + +} + +int main(int argc, char *argv[]) { + return test_main(argc, argv, log_error_tests::run); +} diff --git a/tests/singlestage/singlestage_log_tests.cpp b/tests/singlestage/singlestage_log_tests.cpp new file mode 100644 index 000000000..f03b6bbd1 --- /dev/null +++ b/tests/singlestage/singlestage_log_tests.cpp @@ -0,0 +1,30 @@ +#define SIMDJSON_VERBOSE_LOGGING 1 +#include "simdjson.h" + +#include +#include +#include + +using namespace simdjson; + + +int main() { +#if SIMDJSON_EXCEPTIONS + auto cars_json = R"( [ + { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, + { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, + { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] } + ] )"_padded; + singlestage::parser parser; + singlestage::document doc = parser.iterate(cars_json); + std::vector container; + for(singlestage::object o : doc) { + container.emplace_back(std::string_view(o["make"])); + } + std::cout << "---" << std::endl; + for(std::string& m : container) { + std::cout << m << std::endl; + } +#endif // #if SIMDJSON_EXCEPTIONS + return EXIT_SUCCESS; +} \ No newline at end of file diff --git a/tests/singlestage/singlestage_misc_tests.cpp b/tests/singlestage/singlestage_misc_tests.cpp new file mode 100644 index 000000000..934ea1ce7 --- /dev/null +++ b/tests/singlestage/singlestage_misc_tests.cpp @@ -0,0 +1,631 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace misc_tests { + using namespace std; + bool issue1981_success() { + auto error_phrase = R"(false)"_padded; + TEST_START(); + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(error_phrase).get(doc)); + bool b; + ASSERT_SUCCESS( doc.get_bool().get(b)); + ASSERT_FALSE(b); + TEST_SUCCEED(); + } + + bool issue1981_failure() { + auto error_phrase = R"(falseA)"_padded; + TEST_START(); + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(error_phrase).get(doc)); + bool b; + ASSERT_ERROR( doc.get_bool().get(b), INCORRECT_TYPE); + TEST_SUCCEED(); + } + + bool replacement_char() { + auto fun_phrase = R"( ["I \u2665 Unicode. Even broken \ud800 Unicode." ])"_padded; + std::string_view expected_fun = "I \xe2\x99\xa5 Unicode. Even broken \xef\xbf\xbd Unicode."; + + TEST_START(); + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(fun_phrase).get(doc)); + singlestage::array arr; + ASSERT_SUCCESS( doc.get_array().get(arr)); + std::string_view view; + ASSERT_SUCCESS( arr.at(0).get_string(true).get(view)); + ASSERT_EQUAL(view, expected_fun); + TEST_SUCCEED(); + } + + bool wobbly_tests() { + auto lone_surrogate = R"( "\ud800" )"_padded; + std::string_view expected_lone = "\xed\xa0\x80"; + + + auto fun_phrase = R"( ["I \u2665 Unicode. Even broken \ud800 Unicode." ])"_padded; + std::string_view expected_fun = "I \xe2\x99\xa5 Unicode. Even broken \xed\xa0\x80 Unicode."; + + auto insane_url = R"({"input": "http://example.com/\uDC00\uD834\uDF06\uDC00"} )"_padded; + std::string_view expected_insane = "http://example.com/\xED\xB0\x80\xF0\x9D\x8C\x86\xED\xB0\x80"; + + TEST_START(); + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(lone_surrogate).get(doc)); + std::string_view view; + ASSERT_SUCCESS( doc.get_wobbly_string().get(view)); + ASSERT_EQUAL(view, expected_lone); + + ASSERT_SUCCESS(parser.iterate(fun_phrase).get(doc)); + singlestage::array arr; + ASSERT_SUCCESS( doc.get_array().get(arr)); + ASSERT_SUCCESS( arr.at(0).get_wobbly_string().get(view)); + ASSERT_EQUAL(view, expected_fun); + + ASSERT_SUCCESS(parser.iterate(insane_url).get(doc)); + singlestage::object obj; + ASSERT_SUCCESS( doc.get_object().get(obj)); + ASSERT_SUCCESS( obj["input"].get_wobbly_string().get(view)); + + ASSERT_EQUAL(view, expected_insane); + TEST_SUCCEED(); + } + + bool test_get_value() { + TEST_START(); + singlestage::parser parser; + padded_string json = R"({"a":[[1,null,3.0],["a","b",true],[10000000000,2,3]]})"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::value val; + ASSERT_SUCCESS(doc.get_value().get(val)); + singlestage::object obj; + ASSERT_SUCCESS(val.get_object().get(obj)); + singlestage::array arr; + ASSERT_SUCCESS(obj["a"].get_array().get(arr)); + size_t count{}; + ASSERT_SUCCESS(arr.count_elements().get(count)); + ASSERT_EQUAL(3,count); + TEST_SUCCEED(); + } + + bool issue1661a() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"({"":],"global-groups":[[]}})"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + singlestage::value global_groups; + ASSERT_SUCCESS(doc["global-groups"].get(global_groups)); + singlestage::json_type global_type; + ASSERT_SUCCESS(global_groups.type().get(global_type)); + ASSERT_EQUAL(global_type, singlestage::json_type::array); + TEST_SUCCEED(); + } + + bool doc_ref() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"({"":2,"v":[]})"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + singlestage::document_reference ref(doc); + singlestage::value arr; + ASSERT_SUCCESS(ref["v"].get(arr)); + singlestage::json_type global_type; + ASSERT_SUCCESS(arr.type().get(global_type)); + ASSERT_EQUAL(global_type, singlestage::json_type::array); + TEST_SUCCEED(); + } + + bool issue_uffff() { + TEST_START(); + singlestage::parser parser; + auto json = R"( "wow:\uFFFF" )"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + std::string_view view; + ASSERT_SUCCESS( doc.get_string().get(view)); + ASSERT_EQUAL(view, "wow:\xef\xbf\xbf"); + TEST_SUCCEED(); + } + + bool issue_backslash() { + TEST_START(); + singlestage::parser parser; + auto json = R"( "sc:\\./" )"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + std::string_view view; + ASSERT_SUCCESS( doc.get_string().get(view)); + ASSERT_EQUAL(view, "sc:\\./"); + TEST_SUCCEED(); + } + + bool issue1870() { + TEST_START(); + singlestage::parser parser; + auto json = R"("\uDC00")"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + std::string_view view; + ASSERT_ERROR( doc.get_string().get(view), STRING_ERROR ); + TEST_SUCCEED(); + } + + // Test a surrogate pair with the low surrogate out of range + bool issue1894() { + TEST_START(); + singlestage::parser parser; + auto json = R"("\uD888\u1234")"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + std::string_view view; + ASSERT_ERROR(doc.get_string().get(view), STRING_ERROR); + TEST_SUCCEED(); + } + + bool issue1894toolarge() { + TEST_START(); + singlestage::parser parser; + auto json = R"("\uD888\uE000")"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + std::string_view view; + ASSERT_ERROR(doc.get_string().get(view), STRING_ERROR); + TEST_SUCCEED(); + } + + // Test the smallest surrogate pair, largest surrogate pair, and a surrogate pair in range. + bool issue1894success() { + TEST_START(); + singlestage::parser parser; + auto json = R"("\uD888\uDC00\uD800\uDC00\uDBFF\uDFFF")"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + std::string_view view; + ASSERT_SUCCESS(doc.get_string().get(view)); + ASSERT_EQUAL(view, "\xf0\xb2\x80\x80\xf0\x90\x80\x80\xf4\x8f\xbf\xbf"); + TEST_SUCCEED(); + } + + bool issue1660() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"({"globals":{"a":{"shadowable":[}}}})"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + singlestage::object globals; + ASSERT_SUCCESS(doc["globals"].get(globals)); + for (auto global_field : globals) { + singlestage::value global; + ASSERT_SUCCESS(global_field.value().get(global)); + singlestage::json_type global_type; + ASSERT_SUCCESS(global.type().get(global_type)); + ASSERT_EQUAL(global_type, singlestage::json_type::object); + singlestage::object global_object; + ASSERT_SUCCESS(global.get(global_object)); + singlestage::value shadowable; + ASSERT_SUCCESS(global_object["shadowable"].get(shadowable)); + ASSERT_ERROR(shadowable.get_object(), INCORRECT_TYPE); + singlestage::value badvalue; + auto error = global_object["writable"].get(badvalue); + if(error == SUCCESS) { + return false; + } else { + break; + } + } + TEST_SUCCEED(); + } + + + bool issue1660_with_bool() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"({"globals":{"a":{"shadowable":[}}}})"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + singlestage::object globals; + ASSERT_SUCCESS(doc["globals"].get(globals)); + for (auto global_field : globals) { + singlestage::value global; + ASSERT_SUCCESS(global_field.value().get(global)); + singlestage::json_type global_type; + ASSERT_SUCCESS(global.type().get(global_type)); + ASSERT_EQUAL(global_type, singlestage::json_type::object); + singlestage::object global_object; + ASSERT_SUCCESS(global.get(global_object)); + singlestage::value shadowable; + ASSERT_SUCCESS(global_object["shadowable"].get(shadowable)); + ASSERT_ERROR(shadowable.get_bool(), INCORRECT_TYPE); + singlestage::value badvalue; + auto error = global_object["writable"].get(badvalue); + if(error == SUCCESS) { + return false; + } else { + break; + } + } + TEST_SUCCEED(); + } + + + bool issue1660_with_uint64() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"({"globals":{"a":{"shadowable":[}}}})"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + singlestage::object globals; + ASSERT_SUCCESS(doc["globals"].get(globals)); + for (auto global_field : globals) { + singlestage::value global; + ASSERT_SUCCESS(global_field.value().get(global)); + singlestage::json_type global_type; + ASSERT_SUCCESS(global.type().get(global_type)); + ASSERT_EQUAL(global_type, singlestage::json_type::object); + singlestage::object global_object; + ASSERT_SUCCESS(global.get(global_object)); + singlestage::value shadowable; + ASSERT_SUCCESS(global_object["shadowable"].get(shadowable)); + ASSERT_ERROR(shadowable.get_uint64(), INCORRECT_TYPE); + singlestage::value badvalue; + auto error = global_object["writable"].get(badvalue); + if(error == SUCCESS) { + return false; + } else { + break; + } + } + TEST_SUCCEED(); + } + + + bool issue1660_with_int64() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"({"globals":{"a":{"shadowable":[}}}})"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + singlestage::object globals; + ASSERT_SUCCESS(doc["globals"].get(globals)); + for (auto global_field : globals) { + singlestage::value global; + ASSERT_SUCCESS(global_field.value().get(global)); + singlestage::json_type global_type; + ASSERT_SUCCESS(global.type().get(global_type)); + ASSERT_EQUAL(global_type, singlestage::json_type::object); + singlestage::object global_object; + ASSERT_SUCCESS(global.get(global_object)); + singlestage::value shadowable; + ASSERT_SUCCESS(global_object["shadowable"].get(shadowable)); + ASSERT_ERROR(shadowable.get_int64(), INCORRECT_TYPE); + singlestage::value badvalue; + auto error = global_object["writable"].get(badvalue); + if(error == SUCCESS) { + return false; + } else { + break; + } + } + TEST_SUCCEED(); + } + + bool issue1660_with_double() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"({"globals":{"a":{"shadowable":[}}}})"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + singlestage::object globals; + ASSERT_SUCCESS(doc["globals"].get(globals)); + for (auto global_field : globals) { + singlestage::value global; + ASSERT_SUCCESS(global_field.value().get(global)); + singlestage::json_type global_type; + ASSERT_SUCCESS(global.type().get(global_type)); + ASSERT_EQUAL(global_type, singlestage::json_type::object); + singlestage::object global_object; + ASSERT_SUCCESS(global.get(global_object)); + singlestage::value shadowable; + ASSERT_SUCCESS(global_object["shadowable"].get(shadowable)); + ASSERT_ERROR(shadowable.get_double(), INCORRECT_TYPE); + singlestage::value badvalue; + auto error = global_object["writable"].get(badvalue); + if(error == SUCCESS) { + return false; + } else { + break; + } + } + TEST_SUCCEED(); + } + + + bool issue1660_with_null() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"({"globals":{"a":{"shadowable":[}}}})"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + singlestage::object globals; + ASSERT_SUCCESS(doc["globals"].get(globals)); + for (auto global_field : globals) { + singlestage::value global; + ASSERT_SUCCESS(global_field.value().get(global)); + singlestage::json_type global_type; + ASSERT_SUCCESS(global.type().get(global_type)); + ASSERT_EQUAL(global_type, singlestage::json_type::object); + singlestage::object global_object; + ASSERT_SUCCESS(global.get(global_object)); + singlestage::value shadowable; + ASSERT_SUCCESS(global_object["shadowable"].get(shadowable)); + bool is_null_value; + ASSERT_SUCCESS(shadowable.is_null().get(is_null_value)); + ASSERT_TRUE(!is_null_value); + singlestage::value badvalue; + auto error = global_object["writable"].get(badvalue); + if(error == SUCCESS) { + return false; + } else { + break; + } + } + TEST_SUCCEED(); + } + + + bool issue1660_with_string() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"({"globals":{"a":{"shadowable":[}}}})"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + singlestage::object globals; + ASSERT_SUCCESS(doc["globals"].get(globals)); + for (auto global_field : globals) { + singlestage::value global; + ASSERT_SUCCESS(global_field.value().get(global)); + singlestage::json_type global_type; + ASSERT_SUCCESS(global.type().get(global_type)); + ASSERT_EQUAL(global_type, singlestage::json_type::object); + singlestage::object global_object; + ASSERT_SUCCESS(global.get(global_object)); + singlestage::value shadowable; + ASSERT_SUCCESS(global_object["shadowable"].get(shadowable)); + ASSERT_ERROR(shadowable.get_string(), INCORRECT_TYPE); + singlestage::value badvalue; + auto error = global_object["writable"].get(badvalue); + if(error == SUCCESS) { + return false; + } else { + break; + } + } + TEST_SUCCEED(); + } + bool is_alive_root_array() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"([1,2,3)"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + ASSERT_TRUE(doc.is_alive()); + size_t count{0}; + for(simdjson_unused simdjson_result val : doc) { + ASSERT_ERROR(val.error(), INCOMPLETE_ARRAY_OR_OBJECT); + count++; + } + ASSERT_EQUAL(count, 1); + ASSERT_FALSE(doc.is_alive()); + TEST_SUCCEED(); + } + + bool is_alive_array() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"({"a":[1,2,3})"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + ASSERT_TRUE(doc.is_alive()); + singlestage::object obj; + ASSERT_SUCCESS(doc.get_object().get(obj)); + singlestage::array internal_arr; + ASSERT_SUCCESS(obj["a"].get_array().get(internal_arr)); + size_t count{0}; + for(simdjson_unused simdjson_result val : internal_arr) { + if(count < 3) { + ASSERT_SUCCESS(val.error()); + ASSERT_TRUE(doc.is_alive()); + } else { + ASSERT_ERROR(val.error(), TAPE_ERROR); + ASSERT_FALSE(doc.is_alive()); + } + count++; + } + ASSERT_EQUAL(count, 4); + ASSERT_FALSE(doc.is_alive()); + TEST_SUCCEED(); + } + + bool is_alive_root_object() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"({"a":1)"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + ASSERT_TRUE(doc.is_alive()); + singlestage::object obj; + ASSERT_ERROR(doc.get_object().get(obj), INCOMPLETE_ARRAY_OR_OBJECT); + ASSERT_FALSE(doc.is_alive()); + TEST_SUCCEED(); + } + + bool is_alive_object() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"([{"a":1])"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + ASSERT_TRUE(doc.is_alive()); + singlestage::object obj; + ASSERT_SUCCESS(doc.at(0).get_object().get(obj)); + ASSERT_TRUE(doc.is_alive()); + size_t count{0}; + for(simdjson_unused simdjson_result val : obj) { + if(count == 0) { + ASSERT_SUCCESS(val.error()); + ASSERT_TRUE(doc.is_alive()); + } + if(count == 1) { + ASSERT_ERROR(val.error(), TAPE_ERROR); + ASSERT_FALSE(doc.is_alive()); + } + count++; + } + ASSERT_EQUAL(count, 2); + ASSERT_FALSE(doc.is_alive()); + TEST_SUCCEED(); + } + bool issue1661() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"({"":],"global-groups":[[]}})"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + singlestage::object global_groups; + ASSERT_ERROR(doc["global-groups"].get(global_groups), INCORRECT_TYPE); + singlestage::object globals; + auto error = doc["globals"].get(globals); + if(error == SUCCESS) { return false; } + TEST_SUCCEED(); + } + + simdjson_warn_unused bool big_integer() { + TEST_START(); + simdjson::singlestage::parser parser; + simdjson::padded_string docdata = R"({"value":12321323213213213213213213213211223})"_padded; + simdjson::singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + simdjson::singlestage::object o; + ASSERT_SUCCESS(doc.get_object().get(o)); + string_view token; + ASSERT_SUCCESS(o["value"].raw_json_token().get(token)); + ASSERT_EQUAL(token, "12321323213213213213213213213211223"); + return true; + } + simdjson_warn_unused bool big_integer_in_string() { + TEST_START(); + simdjson::singlestage::parser parser; + simdjson::padded_string docdata = R"({"value":"12321323213213213213213213213211223"})"_padded; + simdjson::singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + simdjson::singlestage::object o; + ASSERT_SUCCESS(doc.get_object().get(o)); + string_view token; + ASSERT_SUCCESS(o["value"].raw_json_token().get(token)); + ASSERT_EQUAL(token, "\"12321323213213213213213213213211223\""); + return true; + } + simdjson_warn_unused bool test_raw_json_token(string_view json, string_view expected_token, int expected_start_index = 0) { + string title("'"); + title.append(json.data(), json.length()); + title += std::string("'"); + padded_string json_padded = json; + SUBTEST(title, test_singlestage_doc(json_padded, [&](auto doc) { + string_view token; + ASSERT_SUCCESS( doc.raw_json_token().get(token) ); + ASSERT_EQUAL( token, expected_token ); + // Validate the text is inside the original buffer + ASSERT_EQUAL( reinterpret_cast(token.data()), reinterpret_cast(&json_padded.data()[expected_start_index])); + return true; + })); + + // Test values + auto json_in_hash = string(R"({"a":)"); + json_in_hash.append(json.data(), json.length()); + json_in_hash += std::string("}"); + json_padded = json_in_hash; + title = std::string("'"); + title.append(json_in_hash.data(), json_in_hash.length()); + title += std::string("'"); + SUBTEST(title, test_singlestage_doc(json_padded, [&](auto doc) { + string_view token; + ASSERT_SUCCESS( doc["a"].raw_json_token().get(token) ); + ASSERT_EQUAL( token, expected_token ); + // Validate the text is inside the original buffer + // Adjust for the {"a": + ASSERT_EQUAL( reinterpret_cast(token.data()), reinterpret_cast(&json_padded.data()[5+expected_start_index])); + return true; + })); + + return true; + } + + bool raw_json_token() { + TEST_START(); + return + test_raw_json_token("{}", "{") && + test_raw_json_token("{ }", "{ ") && + test_raw_json_token("{ \n }", "{ \n ") && + test_raw_json_token(" \n { \n } \n ", "{ \n ", 3) && + test_raw_json_token("[]", "[") && + test_raw_json_token("1", "1") && + test_raw_json_token(" \n 1 \n ", "1 \n ", 3) && + test_raw_json_token("-123.456e-789", "-123.456e-789") && + test_raw_json_token(" \n -123.456e-789 \n ", "-123.456e-789 \n ", 3) && + test_raw_json_token("true", "true") && + test_raw_json_token("false", "false") && + test_raw_json_token("null", "null") && + test_raw_json_token("blah2", "blah2") && + test_raw_json_token("true false", "true ") && + test_raw_json_token("true \n false", "true \n ") && + true; + } + + bool run() { + return + issue1981_success() && + issue1981_failure() && + replacement_char() && + wobbly_tests() && + issue_uffff() && + issue_backslash() && + issue1870() && + issue1894() && + issue1894toolarge() && + issue1894success() && + is_alive_root_array() && + is_alive_root_object() && + is_alive_array() && + is_alive_object() && + test_get_value() && + issue1660_with_uint64() && + issue1660_with_int64() && + issue1660_with_double() && + issue1660_with_null() && + issue1660_with_string() && + issue1660_with_bool() && + issue1661a() && + issue1660() && + issue1661() && + big_integer_in_string() && + big_integer() && + raw_json_token() && + doc_ref() && + true; + } + +} // namespace twitter_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, misc_tests::run); +} diff --git a/tests/singlestage/singlestage_number_in_string_tests.cpp b/tests/singlestage/singlestage_number_in_string_tests.cpp new file mode 100644 index 000000000..9c6e92a6e --- /dev/null +++ b/tests/singlestage/singlestage_number_in_string_tests.cpp @@ -0,0 +1,368 @@ +#include "simdjson.h" +#include "test_singlestage.h" +#include + +using namespace simdjson; + +namespace number_in_string_tests { + const padded_string CRYPTO_JSON = R"( + { + "ticker":{ + "base":"BTC", + "target":"USD", + "price":"443.7807865468", + "volume":"31720.1493969300", + "change":"Infinity", + "markets":[ + { + "market":"bitfinex", + "price":"447.5000000000", + "volume":"10559.5293639000" + }, + { + "market":"bitstamp", + "price":"448.5400000000", + "volume":"11628.2880079300" + }, + { + "market":"btce", + "price":"432.8900000000", + "volume":"8561.0563600000" + } + ] + }, + "timestamp":1399490941, + "timestampstr":"1399490941" + } + )"_padded; + + bool in_document_int64() { + TEST_START(); + auto json = R"( "-11232321" )"_padded; + singlestage::parser parser; + auto doc = parser.iterate(json); + int64_t result; + ASSERT_SUCCESS(doc.get_int64_in_string().get(result)); + ASSERT_EQUAL(result,-11232321); + TEST_SUCCEED(); + } + + bool in_document_uint64() { + TEST_START(); + auto json = R"( "11232321" )"_padded; + singlestage::parser parser; + auto doc = parser.iterate(json); + uint64_t result; + ASSERT_SUCCESS(doc.get_uint64_in_string().get(result)); + ASSERT_EQUAL(result,11232321); + TEST_SUCCEED(); + } + + bool in_document_double() { + TEST_START(); + auto json = R"( "11232321.12" )"_padded; + singlestage::parser parser; + auto doc = parser.iterate(json); + double result; + ASSERT_SUCCESS(doc.get_double_in_string().get(result)); + ASSERT_EQUAL(result,11232321.12); + TEST_SUCCEED(); + } + + bool stream_of_pure_int64() { + TEST_START(); + auto json = R"( 1 2 3 )"_padded; + + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(json).get(stream)); + int document_index = 0; + for (auto doc : stream) { + document_index++; + int64_t result; + ASSERT_SUCCESS(doc.get_int64().get(result)); + ASSERT_EQUAL(result,document_index); + } + ASSERT_EQUAL(3,document_index); + TEST_SUCCEED(); + } + + bool stream_of_direct_int64() { + TEST_START(); + auto json = R"( "1" "2" "3" )"_padded; + + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(json).get(stream)); + int document_index = 0; + for (auto doc : stream) { + document_index++; + int64_t result; + ASSERT_SUCCESS(doc.get_int64_in_string().get(result)); + ASSERT_EQUAL(result,document_index); + } + ASSERT_EQUAL(3,document_index); + TEST_SUCCEED(); + } + + bool stream_of_int64() { + TEST_START(); + auto json = R"( ["1"] ["2"] ["3"] )"_padded; + + singlestage::parser parser; + singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(json).get(stream)); + int document_index = 0; + for (auto doc : stream) { + document_index++; + int64_t result; + ASSERT_SUCCESS(doc.get_array().at(0).get_int64_in_string().get(result)); + ASSERT_EQUAL(result,document_index); + } + ASSERT_EQUAL(3,document_index); + TEST_SUCCEED(); + } + + bool array_double() { + TEST_START(); + auto json = R"(["1.2","2.3","-42.3","2.43442e3", "-1.234e3"])"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + size_t counter{0}; + std::vector expected = {1.2, 2.3, -42.3, 2434.42, -1234}; + double d; + for (auto value : doc) { + ASSERT_SUCCESS(value.get_double_in_string().get(d)); + ASSERT_EQUAL(d,expected[counter++]); + } + TEST_SUCCEED(); + } + + bool array_int() { + TEST_START(); + auto json = R"(["1", "2", "-3", "1000", "-7844", "-9223372036854775807", "9223372036854775807"])"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + size_t counter{0}; + std::vector expected = {1, 2, -3, 1000, -7844, INT64_C(-9223372036854775807), INT64_C(9223372036854775807) }; + int64_t i; + for (auto value : doc) { + ASSERT_SUCCESS(value.get_int64_in_string().get(i)); + ASSERT_EQUAL(i,expected[counter++]); + } + TEST_SUCCEED(); + } + + bool array_unsigned() { + TEST_START(); + auto json = R"(["1", "2", "24", "9000", "156934", "10588030077111859193", "18446744073709551615"])"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + size_t counter{0}; + std::vector expected = {1, 2, 24, 9000, 156934, UINT64_C(10588030077111859193), UINT64_C(18446744073709551615)}; + uint64_t u; + for (auto value : doc) { + ASSERT_SUCCESS(value.get_uint64_in_string().get(u)); + ASSERT_EQUAL(u,expected[counter++]); + } + TEST_SUCCEED(); + } + + bool object() { + TEST_START(); + auto json = R"({"a":"1.2", "b":"-2.342e2", "c":"22", "d":"-112358", "e":"1080", "f":"123456789", "g":"10588030077111859193"})"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + size_t counter{0}; + std::vector expected = {1.2, -234.2, 22, -112358, 1080, 123456789}; + double d; + int64_t i; + uint64_t u; + // Doubles + ASSERT_SUCCESS(doc.find_field("a").get_double_in_string().get(d)); + ASSERT_EQUAL(d,expected[counter++]); + ASSERT_SUCCESS(doc.find_field("b").get_double_in_string().get(d)); + ASSERT_EQUAL(d,expected[counter++]); + // Integers + ASSERT_SUCCESS(doc.find_field("c").get_int64_in_string().get(i)); + ASSERT_EQUAL(i,expected[counter++]); + ASSERT_SUCCESS(doc.find_field("d").get_int64_in_string().get(i)); + ASSERT_EQUAL(i,expected[counter++]); + // Unsigned integers + ASSERT_SUCCESS(doc.find_field("e").get_uint64_in_string().get(u)); + ASSERT_EQUAL(u,expected[counter++]); + ASSERT_SUCCESS(doc.find_field("f").get_uint64_in_string().get(u)); + ASSERT_EQUAL(u,expected[counter++]); + ASSERT_SUCCESS(doc.find_field("g").get_uint64_in_string().get(u)); + ASSERT_EQUAL(u, UINT64_C(10588030077111859193)); + TEST_SUCCEED(); + } + + bool docs() { + TEST_START(); + auto double_doc = R"( "-1.23e1" )"_padded; + auto int_doc = R"( "-243" )"_padded; + auto uint_doc = R"( "212213" )"_padded; + singlestage::parser parser; + singlestage::document doc; + double d; + int64_t i; + uint64_t u; + // Double + ASSERT_SUCCESS(parser.iterate(double_doc).get(doc)); + ASSERT_SUCCESS(doc.get_double_in_string().get(d)); + ASSERT_EQUAL(d,-12.3); + // Integer + ASSERT_SUCCESS(parser.iterate(int_doc).get(doc)); + ASSERT_SUCCESS(doc.get_int64_in_string().get(i)); + ASSERT_EQUAL(i,-243); + // Unsigned integer + ASSERT_SUCCESS(parser.iterate(uint_doc).get(doc)); + ASSERT_SUCCESS(doc.get_uint64_in_string().get(u)); + ASSERT_EQUAL(u,212213); + TEST_SUCCEED(); + } + + bool number_parsing_error() { + TEST_START(); + auto json = R"( ["13.06.54", "1.0e", "2e3r4,,."])"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + size_t counter{0}; + std::string expected[3] = {"13.06.54", "1.0e", "2e3r4,,."}; + for (auto value : doc) { + double d; + std::string_view view; + ASSERT_ERROR(value.get_double_in_string().get(d),NUMBER_ERROR); + ASSERT_SUCCESS(value.get_string().get(view)); + ASSERT_EQUAL(view,expected[counter++]); + } + ASSERT_EQUAL(counter,3); + TEST_SUCCEED(); + } + + bool incorrect_type_error() { + TEST_START(); + auto json = R"( ["e", "i", "pi", "one", "zero"])"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + size_t counter{0}; + std::string expected[5] = {"e", "i", "pi", "one", "zero"}; + for (auto value : doc) { + double d; + std::string_view view; + ASSERT_ERROR(value.get_double_in_string().get(d),INCORRECT_TYPE); + ASSERT_SUCCESS(value.get_string().get(view)); + ASSERT_EQUAL(view,expected[counter++]); + } + ASSERT_EQUAL(counter,5); + TEST_SUCCEED(); + } + + bool json_pointer_test() { + TEST_START(); + auto json = R"( ["12.34", { "a":["3","5.6"], "b":{"c":"1.23e1"} }, ["1", "3.5"] ])"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + std::vector expected = {12.34, 5.6, 12.3, 1, 3.5}; + size_t counter{0}; + double d; + ASSERT_SUCCESS(doc.at_pointer("/0").get_double_in_string().get(d)); + ASSERT_EQUAL(d,expected[counter++]); + ASSERT_SUCCESS(doc.at_pointer("/1/a/1").get_double_in_string().get(d)); + ASSERT_EQUAL(d,expected[counter++]); + ASSERT_SUCCESS(doc.at_pointer("/1/b/c").get_double_in_string().get(d)); + ASSERT_EQUAL(d,expected[counter++]); + ASSERT_SUCCESS(doc.at_pointer("/2/0").get_double_in_string().get(d)); + ASSERT_EQUAL(d,expected[counter++]); + ASSERT_SUCCESS(doc.at_pointer("/2/1").get_double_in_string().get(d)); + ASSERT_EQUAL(d,expected[counter++]); + ASSERT_EQUAL(counter,5); + TEST_SUCCEED(); + } + + bool crypto_timestamp() { + TEST_START(); + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(CRYPTO_JSON).get(doc)); + uint64_t u; + ASSERT_SUCCESS(doc.at_pointer("/timestampstr").get_uint64_in_string().get(u)); + ASSERT_EQUAL(u,1399490941); + TEST_SUCCEED(); + } + + bool crypto_market() { + TEST_START(); + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(CRYPTO_JSON).get(doc)); + singlestage::array markets; + ASSERT_SUCCESS(doc.find_field("ticker").find_field("markets").get_array().get(markets)); + std::string_view expected_views[3] = {"bitfinex", "bitstamp", "btce"}; + double expected_prices[3] = {447.5, 448.54, 432.89}; + double expected_volumes[3] = {10559.5293639, 11628.28800793, 8561.05636}; + size_t counter{0}; + for (auto value : markets) { + std::string_view view; + double price; + double volume; + ASSERT_SUCCESS(value.find_field("market").get_string().get(view)); + ASSERT_EQUAL(view,expected_views[counter]); + ASSERT_SUCCESS(value.find_field("price").get_double_in_string().get(price)); + ASSERT_EQUAL(price,expected_prices[counter]); + ASSERT_SUCCESS(value.find_field("volume").get_double_in_string().get(volume)); + ASSERT_EQUAL(volume,expected_volumes[counter]); + counter++; + } + ASSERT_EQUAL(counter,3); + TEST_SUCCEED(); + } + + bool crypto_infinity() { + TEST_START(); + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(CRYPTO_JSON).get(doc)); + singlestage::value value; + double d; + std::string_view view; + ASSERT_SUCCESS(doc.find_field("ticker").find_field("change").get(value)); + ASSERT_ERROR(value.get_double_in_string().get(d), INCORRECT_TYPE); + ASSERT_SUCCESS(value.get_string().get(view)); + ASSERT_EQUAL(view,"Infinity"); + TEST_SUCCEED(); + } + + bool run() { + return stream_of_pure_int64() && + stream_of_direct_int64() && + stream_of_int64() && + in_document_int64() && + in_document_uint64() && + in_document_double() && + array_double() && + array_int() && + array_unsigned() && + object() && + docs() && + number_parsing_error() && + incorrect_type_error() && + json_pointer_test() && + crypto_timestamp() && + crypto_market() && + crypto_infinity() && + true; + } +} // number_in_string_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, number_in_string_tests::run); +} \ No newline at end of file diff --git a/tests/singlestage/singlestage_number_tests.cpp b/tests/singlestage/singlestage_number_tests.cpp new file mode 100644 index 000000000..366d02e5c --- /dev/null +++ b/tests/singlestage/singlestage_number_tests.cpp @@ -0,0 +1,422 @@ +#include +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace number_tests { + + bool small_integers() { + TEST_START(); + for (int64_t m = 10; m < 20; m++) { + for (int64_t i = -1024; i < 1024; i++) { + if(!test_singlestage(std::to_string(i), + [&](int64_t actual) { + ASSERT_EQUAL(actual, i); + return true; + })) { + return false; + } // if + } // for i + } // for m + return true; + } + + bool powers_of_two() { + TEST_START(); + + // converts the double "expected" to a padded string + auto format_into_padded=[](const double expected) -> padded_string + { + std::vector buf(1024); + const auto n = std::snprintf(buf.data(), + buf.size(), + "%.*e", + std::numeric_limits::max_digits10 - 1, + expected); + const auto nz=static_cast(n); + if (n<0 || nz >= buf.size()) { std::abort(); } + return padded_string(buf.data(), nz); + }; + + for (int i = -1075; i < 1024; ++i) {// large negative values should be zero. + const double expected = std::pow(2, i); + const auto buf=format_into_padded(expected); + std::fflush(nullptr); + if(!test_singlestage(buf, + [&](double actual) { + if(actual!=expected) { + std::cerr << "JSON '" << buf << " parsed to "; + std::fprintf( stderr," %18.18g instead of %18.18g\n", actual, expected); // formatting numbers is easier with printf + SIMDJSON_SHOW_DEFINE(FLT_EVAL_METHOD); + return false; + } + return true; + })) { + return false; + } // if + } // for i + return true; + } + + static const double testing_power_of_ten[] = { + 1e-307, 1e-306, 1e-305, 1e-304, 1e-303, 1e-302, 1e-301, 1e-300, 1e-299, + 1e-298, 1e-297, 1e-296, 1e-295, 1e-294, 1e-293, 1e-292, 1e-291, 1e-290, + 1e-289, 1e-288, 1e-287, 1e-286, 1e-285, 1e-284, 1e-283, 1e-282, 1e-281, + 1e-280, 1e-279, 1e-278, 1e-277, 1e-276, 1e-275, 1e-274, 1e-273, 1e-272, + 1e-271, 1e-270, 1e-269, 1e-268, 1e-267, 1e-266, 1e-265, 1e-264, 1e-263, + 1e-262, 1e-261, 1e-260, 1e-259, 1e-258, 1e-257, 1e-256, 1e-255, 1e-254, + 1e-253, 1e-252, 1e-251, 1e-250, 1e-249, 1e-248, 1e-247, 1e-246, 1e-245, + 1e-244, 1e-243, 1e-242, 1e-241, 1e-240, 1e-239, 1e-238, 1e-237, 1e-236, + 1e-235, 1e-234, 1e-233, 1e-232, 1e-231, 1e-230, 1e-229, 1e-228, 1e-227, + 1e-226, 1e-225, 1e-224, 1e-223, 1e-222, 1e-221, 1e-220, 1e-219, 1e-218, + 1e-217, 1e-216, 1e-215, 1e-214, 1e-213, 1e-212, 1e-211, 1e-210, 1e-209, + 1e-208, 1e-207, 1e-206, 1e-205, 1e-204, 1e-203, 1e-202, 1e-201, 1e-200, + 1e-199, 1e-198, 1e-197, 1e-196, 1e-195, 1e-194, 1e-193, 1e-192, 1e-191, + 1e-190, 1e-189, 1e-188, 1e-187, 1e-186, 1e-185, 1e-184, 1e-183, 1e-182, + 1e-181, 1e-180, 1e-179, 1e-178, 1e-177, 1e-176, 1e-175, 1e-174, 1e-173, + 1e-172, 1e-171, 1e-170, 1e-169, 1e-168, 1e-167, 1e-166, 1e-165, 1e-164, + 1e-163, 1e-162, 1e-161, 1e-160, 1e-159, 1e-158, 1e-157, 1e-156, 1e-155, + 1e-154, 1e-153, 1e-152, 1e-151, 1e-150, 1e-149, 1e-148, 1e-147, 1e-146, + 1e-145, 1e-144, 1e-143, 1e-142, 1e-141, 1e-140, 1e-139, 1e-138, 1e-137, + 1e-136, 1e-135, 1e-134, 1e-133, 1e-132, 1e-131, 1e-130, 1e-129, 1e-128, + 1e-127, 1e-126, 1e-125, 1e-124, 1e-123, 1e-122, 1e-121, 1e-120, 1e-119, + 1e-118, 1e-117, 1e-116, 1e-115, 1e-114, 1e-113, 1e-112, 1e-111, 1e-110, + 1e-109, 1e-108, 1e-107, 1e-106, 1e-105, 1e-104, 1e-103, 1e-102, 1e-101, + 1e-100, 1e-99, 1e-98, 1e-97, 1e-96, 1e-95, 1e-94, 1e-93, 1e-92, + 1e-91, 1e-90, 1e-89, 1e-88, 1e-87, 1e-86, 1e-85, 1e-84, 1e-83, + 1e-82, 1e-81, 1e-80, 1e-79, 1e-78, 1e-77, 1e-76, 1e-75, 1e-74, + 1e-73, 1e-72, 1e-71, 1e-70, 1e-69, 1e-68, 1e-67, 1e-66, 1e-65, + 1e-64, 1e-63, 1e-62, 1e-61, 1e-60, 1e-59, 1e-58, 1e-57, 1e-56, + 1e-55, 1e-54, 1e-53, 1e-52, 1e-51, 1e-50, 1e-49, 1e-48, 1e-47, + 1e-46, 1e-45, 1e-44, 1e-43, 1e-42, 1e-41, 1e-40, 1e-39, 1e-38, + 1e-37, 1e-36, 1e-35, 1e-34, 1e-33, 1e-32, 1e-31, 1e-30, 1e-29, + 1e-28, 1e-27, 1e-26, 1e-25, 1e-24, 1e-23, 1e-22, 1e-21, 1e-20, + 1e-19, 1e-18, 1e-17, 1e-16, 1e-15, 1e-14, 1e-13, 1e-12, 1e-11, + 1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, + 1e-1, 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, + 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, + 1e17, 1e18, 1e19, 1e20, 1e21, 1e22, 1e23, 1e24, 1e25, + 1e26, 1e27, 1e28, 1e29, 1e30, 1e31, 1e32, 1e33, 1e34, + 1e35, 1e36, 1e37, 1e38, 1e39, 1e40, 1e41, 1e42, 1e43, + 1e44, 1e45, 1e46, 1e47, 1e48, 1e49, 1e50, 1e51, 1e52, + 1e53, 1e54, 1e55, 1e56, 1e57, 1e58, 1e59, 1e60, 1e61, + 1e62, 1e63, 1e64, 1e65, 1e66, 1e67, 1e68, 1e69, 1e70, + 1e71, 1e72, 1e73, 1e74, 1e75, 1e76, 1e77, 1e78, 1e79, + 1e80, 1e81, 1e82, 1e83, 1e84, 1e85, 1e86, 1e87, 1e88, + 1e89, 1e90, 1e91, 1e92, 1e93, 1e94, 1e95, 1e96, 1e97, + 1e98, 1e99, 1e100, 1e101, 1e102, 1e103, 1e104, 1e105, 1e106, + 1e107, 1e108, 1e109, 1e110, 1e111, 1e112, 1e113, 1e114, 1e115, + 1e116, 1e117, 1e118, 1e119, 1e120, 1e121, 1e122, 1e123, 1e124, + 1e125, 1e126, 1e127, 1e128, 1e129, 1e130, 1e131, 1e132, 1e133, + 1e134, 1e135, 1e136, 1e137, 1e138, 1e139, 1e140, 1e141, 1e142, + 1e143, 1e144, 1e145, 1e146, 1e147, 1e148, 1e149, 1e150, 1e151, + 1e152, 1e153, 1e154, 1e155, 1e156, 1e157, 1e158, 1e159, 1e160, + 1e161, 1e162, 1e163, 1e164, 1e165, 1e166, 1e167, 1e168, 1e169, + 1e170, 1e171, 1e172, 1e173, 1e174, 1e175, 1e176, 1e177, 1e178, + 1e179, 1e180, 1e181, 1e182, 1e183, 1e184, 1e185, 1e186, 1e187, + 1e188, 1e189, 1e190, 1e191, 1e192, 1e193, 1e194, 1e195, 1e196, + 1e197, 1e198, 1e199, 1e200, 1e201, 1e202, 1e203, 1e204, 1e205, + 1e206, 1e207, 1e208, 1e209, 1e210, 1e211, 1e212, 1e213, 1e214, + 1e215, 1e216, 1e217, 1e218, 1e219, 1e220, 1e221, 1e222, 1e223, + 1e224, 1e225, 1e226, 1e227, 1e228, 1e229, 1e230, 1e231, 1e232, + 1e233, 1e234, 1e235, 1e236, 1e237, 1e238, 1e239, 1e240, 1e241, + 1e242, 1e243, 1e244, 1e245, 1e246, 1e247, 1e248, 1e249, 1e250, + 1e251, 1e252, 1e253, 1e254, 1e255, 1e256, 1e257, 1e258, 1e259, + 1e260, 1e261, 1e262, 1e263, 1e264, 1e265, 1e266, 1e267, 1e268, + 1e269, 1e270, 1e271, 1e272, 1e273, 1e274, 1e275, 1e276, 1e277, + 1e278, 1e279, 1e280, 1e281, 1e282, 1e283, 1e284, 1e285, 1e286, + 1e287, 1e288, 1e289, 1e290, 1e291, 1e292, 1e293, 1e294, 1e295, + 1e296, 1e297, 1e298, 1e299, 1e300, 1e301, 1e302, 1e303, 1e304, + 1e305, 1e306, 1e307, 1e308}; + + + + bool powers_of_ten() { + TEST_START(); + std::vector buf(1024); + + const bool is_pow_correct{1e-308 == std::pow(10,-308)}; + const int start_point = is_pow_correct ? -10000 : -307; + if(!is_pow_correct) { + std::cout << "On your system, the pow function is busted. Sorry about that. " << std::endl; + } + for (int i = start_point; i <= 308; ++i) {// large negative values should be zero. + const size_t n = std::snprintf(buf.data(), buf.size(), "1e%d", i); + if (n >= buf.size()) { std::abort(); } + std::fflush(nullptr); + const double expected = ((i >= -307) ? testing_power_of_ten[i + 307]: std::pow(10, i)); + + if(!test_singlestage(padded_string(buf.data(), n), [&](double actual) { + if(actual!=expected) { + std::cerr << "JSON '" << buf.data() << " parsed to "; + std::fprintf( stderr," %18.18g instead of %18.18g\n", actual, expected); // formatting numbers is easier with printf + SIMDJSON_SHOW_DEFINE(FLT_EVAL_METHOD); + return false; + } + return true; + })) { + return false; + } // if + } // for i + return true; + } + + void github_issue_1273() { + padded_string bad(std::string_view("0.0300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000024000000000000000000000000000000000000000000000000000000000000122978293824")); + simdjson::singlestage::parser parser; + simdjson_unused auto blah=parser.iterate(bad); + double x; + simdjson_unused auto blah2=blah.get(x); + } + + bool issue_1898() { + TEST_START(); + padded_string negative_zero_string(std::string_view("-1e-999")); + simdjson::singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(negative_zero_string).get(doc)); + double x; + ASSERT_SUCCESS(doc.get(x)); + // should be minus 0 + ASSERT_TRUE(std::signbit(x)); + ASSERT_TRUE(x == -0); + TEST_SUCCEED(); + } + + bool old_crashes() { + TEST_START(); + github_issue_1273(); + TEST_SUCCEED(); + } + + + bool is_alive_root_array() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"([1,2,3)"_padded; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + ASSERT_TRUE(doc.is_alive()); + size_t count{0}; + for(simdjson_unused simdjson_result val : doc) { + ASSERT_ERROR(val.error(), INCOMPLETE_ARRAY_OR_OBJECT); + count++; + } + ASSERT_EQUAL(count, 1); + ASSERT_FALSE(doc.is_alive()); + TEST_SUCCEED(); + } + + bool get_number_tests() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999,-9223372036854775807,-9223372036854775808])"_padded; + singlestage::number_type expectedtypes[] = {singlestage::number_type::floating_point_number, + singlestage::number_type::signed_integer, + singlestage::number_type::signed_integer, + singlestage::number_type::floating_point_number, + singlestage::number_type::signed_integer, + singlestage::number_type::unsigned_integer, + singlestage::number_type::signed_integer, + singlestage::number_type::signed_integer + }; + bool is_negative[] = {false, false, false, false, true, false, true, true}; + bool is_integer[] = {false, true, true, false, true, true, true, true}; + + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + singlestage::array arr; + ASSERT_SUCCESS(doc.get_array().get(arr)); + size_t counter{0}; + for(simdjson_result valr : arr) { + singlestage::value val; + ASSERT_SUCCESS(valr.get(val)); + singlestage::number_type nt{}; + ASSERT_SUCCESS(val.get_number_type().get(nt)); + ASSERT_EQUAL(expectedtypes[counter], nt); + singlestage::number num; + ASSERT_SUCCESS(val.get_number().get(num)); + ASSERT_EQUAL(is_negative[counter], val.is_negative()); + bool intvalue{}; + ASSERT_SUCCESS(val.is_integer().get(intvalue)); + ASSERT_EQUAL(is_integer[counter], intvalue); + singlestage::number_type t = num.get_number_type(); + ASSERT_EQUAL(expectedtypes[counter], t); + switch(t) { + case singlestage::number_type::signed_integer: + ASSERT_TRUE(num.is_int64()); + break; + case singlestage::number_type::unsigned_integer: + ASSERT_TRUE(num.is_uint64()); + break; + case singlestage::number_type::floating_point_number: + ASSERT_TRUE(num.is_double()); + break; + } + if(counter == 0) { + ASSERT_EQUAL(num.get_double(), 1.0); + ASSERT_EQUAL((double)num, 1.0); + } else if(counter == 1) { + ASSERT_EQUAL(num.get_int64(), 3); + ASSERT_EQUAL((int64_t)num, 3); + } else if(counter == 2) { + ASSERT_EQUAL(num.get_int64(), 1); + ASSERT_EQUAL((int64_t)num, 1); + } else if(counter == 3) { + ASSERT_EQUAL(num.get_double(), 3.1415); + ASSERT_EQUAL((double)num, 3.1415); + } else if(counter == 4) { + ASSERT_EQUAL(num.get_int64(), -13231232); + ASSERT_EQUAL((int64_t)num, -13231232); + } else if(counter == 5) { + ASSERT_EQUAL(num.get_uint64(), UINT64_C(9999999999999999999)); + ASSERT_EQUAL((uint64_t)num, UINT64_C(9999999999999999999)); + } + counter++; + } + TEST_SUCCEED(); + } + + bool issue1878() { + TEST_START(); + singlestage::parser parser; + auto json = R"(123_abc)"_padded; + for (char ch : {'_', '%', 'z', '&', '\\', '/', '*'}) { + json.data()[3] = ch; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ASSERT_ERROR(doc.get_int64().error(), NUMBER_ERROR); + } + for (char ch : {'[', ']', '{', '}', ',', ' '}) { + json.data()[3] = ch; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ASSERT_ERROR(doc.get_int64().error(), TRAILING_CONTENT); + } + for (char ch : {'_', '%', 'z', '&', '\\', '/', '*'}) { + json.data()[3] = ch; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ASSERT_ERROR(doc.get_uint64().error(), NUMBER_ERROR); + } + for (char ch : {'[', ']', '{', '}', ',', ' '}) { + json.data()[3] = ch; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ASSERT_ERROR(doc.get_uint64().error(), TRAILING_CONTENT); + } + for (char ch : {'_', '%', 'z', '&', '\\', '/', '*'}) { + json.data()[3] = ch; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ASSERT_ERROR(doc.get_double().error(), NUMBER_ERROR); + } + for (char ch : {'[', ']', '{', '}', ',', ' '}) { + json.data()[3] = ch; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ASSERT_ERROR(doc.get_double().error(), TRAILING_CONTENT); + } + for (char ch : {'_', '%', 'z', '&', '\\', '/', '*'}) { + json.data()[3] = ch; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ASSERT_ERROR(doc.get_number().error(), NUMBER_ERROR); + } + for (char ch : {'[', ']', '{', '}', ',', ' '}) { + json.data()[3] = ch; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ASSERT_ERROR(doc.get_number().error(), TRAILING_CONTENT); + } + TEST_SUCCEED(); + } + + + + bool get_root_number_tests() { + TEST_START(); + singlestage::parser parser; + singlestage::document doc; + padded_string docdata; + singlestage::number number; + singlestage::number_type nt{}; + + bool intvalue{}; + + docdata = R"(1.0)"_padded; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + ASSERT_FALSE(doc.is_negative()); + ASSERT_SUCCESS(doc.get_number_type().get(nt)); + ASSERT_EQUAL(nt, singlestage::number_type::floating_point_number); + ASSERT_SUCCESS(doc.is_integer().get(intvalue)); + ASSERT_SUCCESS(doc.get_number().get(number)); + ASSERT_FALSE(intvalue); + ASSERT_EQUAL(number.get_number_type(), singlestage::number_type::floating_point_number); + ASSERT_EQUAL(number.get_double(), 1.0); + + docdata = R"(-3.132321)"_padded; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + ASSERT_TRUE(doc.is_negative()); + ASSERT_SUCCESS(doc.get_number_type().get(nt)); + ASSERT_EQUAL(nt, singlestage::number_type::floating_point_number); + ASSERT_SUCCESS(doc.is_integer().get(intvalue)); + ASSERT_SUCCESS(doc.get_number().get(number)); + ASSERT_FALSE(intvalue); + ASSERT_EQUAL(number.get_number_type(), singlestage::number_type::floating_point_number); + ASSERT_EQUAL(number.get_double(), -3.132321); + + docdata = R"(1233)"_padded; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + ASSERT_FALSE(doc.is_negative()); + ASSERT_SUCCESS(doc.get_number_type().get(nt)); + ASSERT_EQUAL(nt, singlestage::number_type::signed_integer); + ASSERT_SUCCESS(doc.is_integer().get(intvalue)); + ASSERT_SUCCESS(doc.get_number().get(number)); + ASSERT_TRUE(intvalue); + ASSERT_EQUAL(number.get_number_type(), singlestage::number_type::signed_integer); + ASSERT_EQUAL(number.get_int64(), 1233); + + docdata = R"(9999999999999999999)"_padded; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + ASSERT_FALSE(doc.is_negative()); + ASSERT_SUCCESS(doc.get_number_type().get(nt)); + ASSERT_EQUAL(nt, singlestage::number_type::unsigned_integer); + ASSERT_SUCCESS(doc.is_integer().get(intvalue)); + ASSERT_SUCCESS(doc.get_number().get(number)); + ASSERT_TRUE(intvalue); + ASSERT_EQUAL(number.get_number_type(), singlestage::number_type::unsigned_integer); + ASSERT_EQUAL(number.get_uint64(), UINT64_C(9999999999999999999)); + + TEST_SUCCEED(); + } + bool issue2017() { + TEST_START(); + singlestage::parser parser; + singlestage::document doc; + padded_string docdata = R"({"score":0.8825149536132812})"_padded; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + double score; + ASSERT_SUCCESS(doc["score"].get_double().get(score)); + ASSERT_EQUAL(score, 0.8825149536132812); + TEST_SUCCEED(); + } + + bool run() { + return issue2017() && + issue_1898() && + issue1878() && + get_root_number_tests() && + get_number_tests()&& + small_integers() && + powers_of_two() && + powers_of_ten() && + old_crashes(); + } + +} // namespace number_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, number_tests::run); +} diff --git a/tests/singlestage/singlestage_object_error_tests.cpp b/tests/singlestage/singlestage_object_error_tests.cpp new file mode 100644 index 000000000..14f8d8feb --- /dev/null +++ b/tests/singlestage/singlestage_object_error_tests.cpp @@ -0,0 +1,573 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace object_error_tests { + using namespace std; + + template + bool assert_iterate_object(T &&object, const char **expected_key, V *expected, size_t N, simdjson::error_code *expected_error, size_t N2) { + size_t count = 0; + for (auto field : object) { + V actual{}; + auto actual_error = field.value().get(actual); + if (count >= N) { + ASSERT((count - N) < N2, "Extra error reported"); + ASSERT_ERROR(actual_error, expected_error[count - N]); + } else { + ASSERT_SUCCESS(actual_error); + ASSERT_EQUAL(field.key().value_unsafe(), expected_key[count]); + ASSERT_EQUAL(actual, expected[count]); + } + count++; + } + ASSERT_EQUAL(count, N+N2); + return true; + } + + template + bool assert_iterate_object(T &&object, const char *(&&expected_key)[N], V (&&expected)[N], simdjson::error_code (&&expected_error)[N2]) { + return assert_iterate_object(std::forward(object), expected_key, expected, N, expected_error, N2); + } + + template + bool assert_iterate_object(T &&object, simdjson::error_code (&&expected_error)[N2]) { + return assert_iterate_object(std::forward(object), nullptr, nullptr, 0, expected_error, N2); + } + + template + bool assert_iterate_object(T &&object, const char *(&&expected_key)[N], V (&&expected)[N]) { + return assert_iterate_object(std::forward(object), expected_key, expected, N, nullptr, 0); + } + + bool object_iterate_error() { + TEST_START(); + SINGLESTAGE_SUBTEST("missing colon", R"({ "a" 1, "b": 2 })", assert_iterate_object(doc.get_object(), { TAPE_ERROR })); + SINGLESTAGE_SUBTEST("missing key ", R"({ : 1, "b": 2 })", assert_iterate_object(doc.get_object(), { TAPE_ERROR })); + SINGLESTAGE_SUBTEST("missing value", R"({ "a": , "b": 2 })", assert_iterate_object(doc.get_object(), { INCORRECT_TYPE, TAPE_ERROR })); + SINGLESTAGE_SUBTEST("missing comma", R"({ "a": 1 "b": 2 })", assert_iterate_object(doc.get_object(), { "a" }, { int64_t(1) }, { TAPE_ERROR })); + TEST_SUCCEED(); + } + bool object_iterate_wrong_key_type_error() { + TEST_START(); + SINGLESTAGE_SUBTEST("wrong key type", R"({ 1: 1, "b": 2 })", assert_iterate_object(doc.get_object(), { TAPE_ERROR })); + SINGLESTAGE_SUBTEST("wrong key type", R"({ true: 1, "b": 2 })", assert_iterate_object(doc.get_object(), { TAPE_ERROR })); + SINGLESTAGE_SUBTEST("wrong key type", R"({ false: 1, "b": 2 })", assert_iterate_object(doc.get_object(), { TAPE_ERROR })); + SINGLESTAGE_SUBTEST("wrong key type", R"({ null: 1, "b": 2 })", assert_iterate_object(doc.get_object(), { TAPE_ERROR })); + SINGLESTAGE_SUBTEST("wrong key type", R"({ []: 1, "b": 2 })", assert_iterate_object(doc.get_object(), { TAPE_ERROR })); + SINGLESTAGE_SUBTEST("wrong key type", R"({ {}: 1, "b": 2 })", assert_iterate_object(doc.get_object(), { TAPE_ERROR })); + TEST_SUCCEED(); + } + bool object_iterate_unclosed_error() { + TEST_START(); + SINGLESTAGE_SUBTEST("unclosed", R"({ "a": 1, )", assert_iterate_object(doc.get_object(), { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed", R"({ "a": 1 )", assert_iterate_object(doc.get_object(), { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed", R"({ "a": )", assert_iterate_object(doc.get_object(), { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed", R"({ "a" )", assert_iterate_object(doc.get_object(), { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed", R"({ )", assert_iterate_object(doc.get_object(), { INCOMPLETE_ARRAY_OR_OBJECT })); + TEST_SUCCEED(); + } + bool object_iterate_incomplete_error() { + TEST_START(); + SINGLESTAGE_SUBTEST("unclosed", R"({ "x": { "a": 1, })", assert_iterate_object(doc.get_object(), { "a" }, { int64_t(1) }, { TAPE_ERROR })); + SINGLESTAGE_SUBTEST("unclosed", R"({ "x": { "a": 1 })", assert_iterate_object(doc.get_object(), { "a" }, { int64_t(1) }, { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed", R"({ "x": { "a": })", assert_iterate_object(doc.get_object(), { INCORRECT_TYPE, INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed", R"({ "x": { "a" })", assert_iterate_object(doc.get_object(), { INCOMPLETE_ARRAY_OR_OBJECT })); + SINGLESTAGE_SUBTEST("unclosed", R"({ "x": { })", assert_iterate_object(doc.get_object(), { INCOMPLETE_ARRAY_OR_OBJECT })); + TEST_SUCCEED(); + } + + bool object_lookup_error() { + TEST_START(); + SINGLESTAGE_SUBTEST("missing colon", R"({ "a" 1, "b": 2 })", assert_error(doc["a"], TAPE_ERROR)); + SINGLESTAGE_SUBTEST("missing key ", R"({ : 1, "b": 2 })", assert_error(doc["a"], TAPE_ERROR)); + SINGLESTAGE_SUBTEST("missing value", R"({ "a": , "b": 2 })", assert_success(doc["a"])); + SINGLESTAGE_SUBTEST("missing comma", R"({ "a": 1 "b": 2 })", assert_success(doc["a"])); + TEST_SUCCEED(); + } + bool object_lookup_unclosed_error() { + TEST_START(); + #if SIMDJSON_CHECK_EOF + SINGLESTAGE_SUBTEST("unclosed", R"({ "a": )", assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT)); + #else + SINGLESTAGE_SUBTEST("unclosed", R"({ "a": )", assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT)); + #endif + SINGLESTAGE_SUBTEST("unclosed", R"({ "a" )", assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT)); + SINGLESTAGE_SUBTEST("unclosed", R"({ )", assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT)); + TEST_SUCCEED(); + } + + bool object_lookup_miss_error() { + TEST_START(); + SINGLESTAGE_SUBTEST("missing colon", R"({ "a" 1, "b": 2 })", assert_error(doc["b"], TAPE_ERROR)); + SINGLESTAGE_SUBTEST("missing key ", R"({ : 1, "b": 2 })", assert_error(doc["b"], TAPE_ERROR)); + SINGLESTAGE_SUBTEST("missing value", R"({ "a": , "b": 2 })", assert_error(doc["b"], TAPE_ERROR)); + SINGLESTAGE_SUBTEST("missing comma", R"({ "a": 1 "b": 2 })", assert_error(doc["b"], TAPE_ERROR)); + TEST_SUCCEED(); + } + bool object_lookup_miss_wrong_key_type_error() { + TEST_START(); + SINGLESTAGE_SUBTEST("wrong key type", R"({ 1: 1, "b": 2 })", assert_error(doc["b"], TAPE_ERROR)); + SINGLESTAGE_SUBTEST("wrong key type", R"({ true: 1, "b": 2 })", assert_error(doc["b"], TAPE_ERROR)); + SINGLESTAGE_SUBTEST("wrong key type", R"({ false: 1, "b": 2 })", assert_error(doc["b"], TAPE_ERROR)); + SINGLESTAGE_SUBTEST("wrong key type", R"({ null: 1, "b": 2 })", assert_error(doc["b"], TAPE_ERROR)); + SINGLESTAGE_SUBTEST("wrong key type", R"({ []: 1, "b": 2 })", assert_error(doc["b"], TAPE_ERROR)); + SINGLESTAGE_SUBTEST("wrong key type", R"({ {}: 1, "b": 2 })", assert_error(doc["b"], TAPE_ERROR)); + TEST_SUCCEED(); + } + bool object_lookup_miss_unclosed_error() { + TEST_START(); + SINGLESTAGE_SUBTEST("unclosed", R"({ "a": 1, )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT)); + // TODO These next two pass the user a value that may run past the end of the buffer if they aren't careful. + // In particular, if the padding is decorated with the wrong values, we could cause overrun! + SINGLESTAGE_SUBTEST("unclosed", R"({ "a": 1 )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT)); + SINGLESTAGE_SUBTEST("unclosed", R"({ "a": )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT)); + SINGLESTAGE_SUBTEST("unclosed", R"({ "a" )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT)); + SINGLESTAGE_SUBTEST("unclosed", R"({ )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT)); + TEST_SUCCEED(); + } + bool object_lookup_miss_next_error() { + TEST_START(); + SINGLESTAGE_SUBTEST("missing comma", R"({ "a": 1 "b": 2 })", ([&]() { + auto obj = doc.get_object(); + return assert_result(obj["a"], int64_t(1)) && assert_error(obj["b"], TAPE_ERROR); + })()); + TEST_SUCCEED(); + } + +#ifdef SIMDJSON_DEVELOPMENT_CHECKS + bool out_of_order_object_iteration_error() { + TEST_START(); + auto json = R"([ { "x": 1, "y": 2 } ])"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + for (auto element : doc) { + auto obj = element.get_object(); + for (auto field : obj) { ASSERT_SUCCESS(field); } + ASSERT_ITERATE_ERROR( obj, OUT_OF_ORDER_ITERATION ); + } + return true; + })); + SUBTEST("object", test_singlestage_doc(json, [&](auto doc) { + for (auto element : doc) { + singlestage::object obj; + ASSERT_SUCCESS( element.get(obj) ); + for (auto field : obj) { ASSERT_SUCCESS(field); } + ASSERT_ITERATE_ERROR( obj, OUT_OF_ORDER_ITERATION ); + } + return true; + })); + TEST_SUCCEED(); + } + + bool out_of_order_top_level_object_iteration_error() { + TEST_START(); + auto json = R"({ "x": 1, "y": 2 })"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + auto obj = doc.get_object(); + for (auto field : obj) { ASSERT_SUCCESS(field); } + ASSERT_ITERATE_ERROR( obj, OUT_OF_ORDER_ITERATION ); + return true; + })); + SUBTEST("object", test_singlestage_doc(json, [&](auto doc) { + singlestage::object obj; + ASSERT_SUCCESS( doc.get(obj) ); + for (auto field : obj) { ASSERT_SUCCESS(field); } + ASSERT_ITERATE_ERROR( obj, OUT_OF_ORDER_ITERATION ); + return true; + })); + TEST_SUCCEED(); + } + + bool out_of_order_object_index_child_error() { + TEST_START(); + auto json = R"([ { "x": 1, "y": 2 } ])"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + simdjson_result obj; + for (auto element : doc) { + obj = element.get_object(); + for (auto field : obj) { ASSERT_SUCCESS(field); } + } + ASSERT_ERROR( obj["x"], OUT_OF_ORDER_ITERATION ); + return true; + })); + SUBTEST("object", test_singlestage_doc(json, [&](auto doc) { + singlestage::object obj; + for (auto element : doc) { + ASSERT_SUCCESS( element.get(obj) ); + for (auto field : obj) { ASSERT_SUCCESS(field); } + } + ASSERT_ERROR( obj["x"], OUT_OF_ORDER_ITERATION ); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + simdjson_result obj; + for (auto element : doc) { + obj = element; + ASSERT_SUCCESS( obj["x"] ); + } + ASSERT_ERROR( obj["x"], OUT_OF_ORDER_ITERATION ); + return true; + })); + SUBTEST("value", test_singlestage_doc(json, [&](auto doc) { + singlestage::value obj; + for (auto element : doc) { + ASSERT_SUCCESS( element.get(obj) ); + ASSERT_SUCCESS( obj["x"] ); + } + ASSERT_ERROR( obj["x"], OUT_OF_ORDER_ITERATION ); + return true; + })); + TEST_SUCCEED(); + } + + bool out_of_order_object_index_sibling_error() { + TEST_START(); + auto json = R"([ { "x": 0, "y": 2 }, { "x": 1, "y": 4 } ])"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + simdjson_result last_obj; + uint64_t i = 0; + for (auto element : doc) { + auto obj = element.get_object(); + uint64_t x; + ASSERT_SUCCESS(obj["x"].get(x)); + ASSERT_EQUAL(x, i); + if (i > 0) { + ASSERT_ERROR(last_obj["x"].get(x), OUT_OF_ORDER_ITERATION); + break; + } + last_obj = obj; + i++; + } + return true; + })); + + SUBTEST("object", test_singlestage_doc(json, [&](auto doc) { + singlestage::object last_obj; + uint64_t i = 0; + for (auto element : doc) { + singlestage::object obj; + ASSERT_SUCCESS( element.get_object().get(obj) ); + uint64_t x; + ASSERT_SUCCESS( obj["x"].get(x) ); + ASSERT_EQUAL(x, i); + if (i > 0) { + ASSERT_ERROR(last_obj["x"].get(x), OUT_OF_ORDER_ITERATION); + break; + } + last_obj = obj; + i++; + } + return true; + })); + + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + simdjson_result last_obj; + uint64_t i = 0; + for (auto element : doc) { + auto obj = element; + uint64_t x; + ASSERT_SUCCESS(obj["x"].get(x)); + ASSERT_EQUAL(x, i); + if (i > 0) { + ASSERT_ERROR(last_obj["x"].get(x), OUT_OF_ORDER_ITERATION); + break; + } + last_obj = obj; + i++; + } + return true; + })); + + SUBTEST("value", test_singlestage_doc(json, [&](auto doc) { + singlestage::value last_obj; + uint64_t i = 0; + for (auto element : doc) { + singlestage::value obj; + ASSERT_SUCCESS( element.get(obj) ); + uint64_t x; + ASSERT_SUCCESS( obj["x"].get(x) ); + ASSERT_EQUAL(x, i); + if (i > 0) { + ASSERT_ERROR(last_obj["x"].get(x), OUT_OF_ORDER_ITERATION); + break; + } + last_obj = obj; + i++; + } + return true; + })); + TEST_SUCCEED(); + } + + bool out_of_order_object_find_field_child_error() { + TEST_START(); + auto json = R"([ { "x": 1, "y": 2 } ])"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + simdjson_result obj; + for (auto element : doc) { + obj = element.get_object(); + for (auto field : obj) { ASSERT_SUCCESS(field); } + } + ASSERT_ERROR( obj.find_field("x"), OUT_OF_ORDER_ITERATION ); + return true; + })); + SUBTEST("object", test_singlestage_doc(json, [&](auto doc) { + singlestage::object obj; + for (auto element : doc) { + ASSERT_SUCCESS( element.get(obj) ); + for (auto field : obj) { ASSERT_SUCCESS(field); } + } + ASSERT_ERROR( obj.find_field("x"), OUT_OF_ORDER_ITERATION ); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + simdjson_result obj; + for (auto element : doc) { + obj = element; + ASSERT_SUCCESS( obj.find_field("x") ); + } + ASSERT_ERROR( obj.find_field("x"), OUT_OF_ORDER_ITERATION ); + return true; + })); + SUBTEST("value", test_singlestage_doc(json, [&](auto doc) { + singlestage::value obj; + for (auto element : doc) { + ASSERT_SUCCESS( element.get(obj) ); + ASSERT_SUCCESS( obj.find_field("x") ); + } + ASSERT_ERROR( obj.find_field("x"), OUT_OF_ORDER_ITERATION ); + return true; + })); + TEST_SUCCEED(); + } + + bool out_of_order_object_find_field_sibling_error() { + TEST_START(); + auto json = R"([ { "x": 0, "y": 2 }, { "x": 1, "y": 4 } ])"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + simdjson_result last_obj; + uint64_t i = 0; + for (auto element : doc) { + auto obj = element.get_object(); + uint64_t x; + ASSERT_SUCCESS(obj.find_field("x").get(x)); + ASSERT_EQUAL(x, i); + if (i > 0) { + ASSERT_ERROR(last_obj.find_field("x").get(x), OUT_OF_ORDER_ITERATION); + break; + } + last_obj = obj; + i++; + } + return true; + })); + + SUBTEST("object", test_singlestage_doc(json, [&](auto doc) { + singlestage::object last_obj; + uint64_t i = 0; + for (auto element : doc) { + singlestage::object obj; + ASSERT_SUCCESS( element.get_object().get(obj) ); + uint64_t x; + ASSERT_SUCCESS( obj.find_field("x").get(x) ); + ASSERT_EQUAL(x, i); + if (i > 0) { + ASSERT_ERROR(last_obj.find_field("x").get(x), OUT_OF_ORDER_ITERATION); + break; + } + last_obj = obj; + i++; + } + return true; + })); + + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + simdjson_result last_obj; + uint64_t i = 0; + for (auto element : doc) { + auto obj = element; + uint64_t x; + ASSERT_SUCCESS(obj.find_field("x").get(x)); + ASSERT_EQUAL(x, i); + if (i > 0) { + ASSERT_ERROR(last_obj.find_field("x").get(x), OUT_OF_ORDER_ITERATION); + break; + } + last_obj = obj; + i++; + } + return true; + })); + + SUBTEST("value", test_singlestage_doc(json, [&](auto doc) { + singlestage::value last_obj; + uint64_t i = 0; + for (auto element : doc) { + singlestage::value obj; + ASSERT_SUCCESS( element.get(obj) ); + uint64_t x; + ASSERT_SUCCESS( obj.find_field("x").get(x) ); + ASSERT_EQUAL(x, i); + if (i > 0) { + ASSERT_ERROR(last_obj.find_field("x").get(x), OUT_OF_ORDER_ITERATION); + break; + } + last_obj = obj; + i++; + } + return true; + })); + TEST_SUCCEED(); + } + + bool out_of_order_object_find_field_unordered_child_error() { + TEST_START(); + auto json = R"([ { "x": 1, "y": 2 } ])"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + simdjson_result obj; + for (auto element : doc) { + obj = element.get_object(); + for (auto field : obj) { ASSERT_SUCCESS(field); } + } + ASSERT_ERROR( obj.find_field_unordered("x"), OUT_OF_ORDER_ITERATION ); + return true; + })); + SUBTEST("object", test_singlestage_doc(json, [&](auto doc) { + singlestage::object obj; + for (auto element : doc) { + ASSERT_SUCCESS( element.get(obj) ); + for (auto field : obj) { ASSERT_SUCCESS(field); } + } + ASSERT_ERROR( obj.find_field_unordered("x"), OUT_OF_ORDER_ITERATION ); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + simdjson_result obj; + for (auto element : doc) { + obj = element; + ASSERT_SUCCESS( obj.find_field_unordered("x") ); + } + ASSERT_ERROR( obj.find_field_unordered("x"), OUT_OF_ORDER_ITERATION ); + return true; + })); + SUBTEST("value", test_singlestage_doc(json, [&](auto doc) { + singlestage::value obj; + for (auto element : doc) { + ASSERT_SUCCESS( element.get(obj) ); + ASSERT_SUCCESS( obj.find_field_unordered("x") ); + } + ASSERT_ERROR( obj.find_field_unordered("x"), OUT_OF_ORDER_ITERATION ); + return true; + })); + TEST_SUCCEED(); + } + + bool out_of_order_object_find_field_unordered_sibling_error() { + TEST_START(); + auto json = R"([ { "x": 0, "y": 2 }, { "x": 1, "y": 4 } ])"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + simdjson_result last_obj; + uint64_t i = 0; + for (auto element : doc) { + auto obj = element.get_object(); + uint64_t x; + ASSERT_SUCCESS(obj.find_field_unordered("x").get(x)); + ASSERT_EQUAL(x, i); + if (i > 0) { + ASSERT_ERROR(last_obj.find_field_unordered("x").get(x), OUT_OF_ORDER_ITERATION); + break; + } + last_obj = obj; + i++; + } + return true; + })); + + SUBTEST("object", test_singlestage_doc(json, [&](auto doc) { + singlestage::object last_obj; + uint64_t i = 0; + for (auto element : doc) { + singlestage::object obj; + ASSERT_SUCCESS( element.get_object().get(obj) ); + uint64_t x; + ASSERT_SUCCESS( obj.find_field_unordered("x").get(x) ); + ASSERT_EQUAL(x, i); + if (i > 0) { + ASSERT_ERROR(last_obj.find_field_unordered("x").get(x), OUT_OF_ORDER_ITERATION); + break; + } + last_obj = obj; + i++; + } + return true; + })); + + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc) { + simdjson_result last_obj; + uint64_t i = 0; + for (auto element : doc) { + auto obj = element; + uint64_t x; + ASSERT_SUCCESS(obj.find_field_unordered("x").get(x)); + ASSERT_EQUAL(x, i); + if (i > 0) { + ASSERT_ERROR(last_obj.find_field_unordered("x").get(x), OUT_OF_ORDER_ITERATION); + break; + } + last_obj = obj; + i++; + } + return true; + })); + + SUBTEST("value", test_singlestage_doc(json, [&](auto doc) { + singlestage::value last_obj; + uint64_t i = 0; + for (auto element : doc) { + singlestage::value obj; + ASSERT_SUCCESS( element.get(obj) ); + uint64_t x; + ASSERT_SUCCESS( obj.find_field_unordered("x").get(x) ); + ASSERT_EQUAL(x, i); + if (i > 0) { + ASSERT_ERROR(last_obj.find_field_unordered("x").get(x), OUT_OF_ORDER_ITERATION); + break; + } + last_obj = obj; + i++; + } + return true; + })); + TEST_SUCCEED(); + } +#endif + + bool run() { + return + object_iterate_error() && + object_iterate_wrong_key_type_error() && + object_iterate_unclosed_error() && + object_lookup_error() && + object_lookup_unclosed_error() && + object_lookup_miss_error() && + object_lookup_miss_unclosed_error() && + object_lookup_miss_wrong_key_type_error() && + object_lookup_miss_next_error() && +#ifdef SIMDJSON_DEVELOPMENT_CHECKS + out_of_order_object_iteration_error() && + out_of_order_top_level_object_iteration_error() && + out_of_order_object_index_child_error() && + out_of_order_object_index_sibling_error() && + out_of_order_object_find_field_child_error() && + out_of_order_object_find_field_sibling_error() && + out_of_order_object_find_field_unordered_child_error() && + out_of_order_object_find_field_unordered_sibling_error() && +#endif + true; + } +} // namespace object_error_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, object_error_tests::run); +} diff --git a/tests/singlestage/singlestage_object_find_field_tests.cpp b/tests/singlestage/singlestage_object_find_field_tests.cpp new file mode 100644 index 000000000..a3f9c62fe --- /dev/null +++ b/tests/singlestage/singlestage_object_find_field_tests.cpp @@ -0,0 +1,191 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace object_tests { + using namespace std; + using simdjson::singlestage::json_type; + + bool object_find_field_unordered() { + TEST_START(); + auto json = R"({ "a": 1, "b": 2, "c/d": 3})"_padded; + SUBTEST("singlestage::object", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_SUCCESS( doc_result.get(object) ); + + ASSERT_EQUAL( object.find_field_unordered("a").get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( object.find_field_unordered("b").get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( object.find_field_unordered("c/d").get_uint64().value_unsafe(), 3 ); + + ASSERT_EQUAL( object.find_field_unordered("a").get_uint64().value_unsafe(), 1 ); + ASSERT_ERROR( object.find_field_unordered("d"), NO_SUCH_FIELD ); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + simdjson_result object; + object = doc_result.get_object(); + + ASSERT_EQUAL( object.find_field_unordered("a").get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( object.find_field_unordered("b").get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( object.find_field_unordered("c/d").get_uint64().value_unsafe(), 3 ); + + ASSERT_EQUAL( object.find_field_unordered("a").get_uint64().value_unsafe(), 1 ); + ASSERT_ERROR( object.find_field_unordered("d"), NO_SUCH_FIELD ); + return true; + })); + TEST_SUCCEED(); + } + + bool document_object_find_field_unordered() { + TEST_START(); + auto json = R"({ "a": 1, "b": 2, "c/d": 3})"_padded; + SUBTEST("singlestage::document", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::document doc; + ASSERT_SUCCESS( std::move(doc_result).get(doc) ); + ASSERT_EQUAL( doc.find_field_unordered("a").get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( doc.find_field_unordered("b").get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( doc.find_field_unordered("c/d").get_uint64().value_unsafe(), 3 ); + + ASSERT_EQUAL( doc.find_field_unordered("a").get_uint64().value_unsafe(), 1 ); + ASSERT_ERROR( doc.find_field_unordered("d"), NO_SUCH_FIELD ); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + ASSERT_EQUAL( doc_result.find_field_unordered("a").get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( doc_result.find_field_unordered("b").get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( doc_result.find_field_unordered("c/d").get_uint64().value_unsafe(), 3 ); + + ASSERT_EQUAL( doc_result.find_field_unordered("a").get_uint64().value_unsafe(), 1 ); + ASSERT_ERROR( doc_result.find_field_unordered("d"), NO_SUCH_FIELD ); + return true; + })); + TEST_SUCCEED(); + } + + bool value_object_find_field_unordered() { + TEST_START(); + auto json = R"({ "outer": { "a": 1, "b": 2, "c/d": 3 } })"_padded; + SUBTEST("singlestage::value", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::value object; + ASSERT_SUCCESS( doc_result.find_field_unordered("outer").get(object) ); + ASSERT_EQUAL( object.find_field_unordered("a").get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( object.find_field_unordered("b").get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( object.find_field_unordered("c/d").get_uint64().value_unsafe(), 3 ); + + ASSERT_EQUAL( object.find_field_unordered("a").get_uint64().value_unsafe(), 1 ); + ASSERT_ERROR( object.find_field_unordered("d"), NO_SUCH_FIELD ); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + simdjson_result object = doc_result.find_field_unordered("outer"); + ASSERT_EQUAL( object.find_field_unordered("a").get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( object.find_field_unordered("b").get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( object.find_field_unordered("c/d").get_uint64().value_unsafe(), 3 ); + + ASSERT_EQUAL( object.find_field_unordered("a").get_uint64().value_unsafe(), 1 ); + ASSERT_ERROR( object.find_field_unordered("d"), NO_SUCH_FIELD ); + return true; + })); + TEST_SUCCEED(); + } + + bool object_find_field() { + TEST_START(); + auto json = R"({ "a": 1, "b": 2, "c/d": 3})"_padded; + SUBTEST("singlestage::object", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_SUCCESS( doc_result.get(object) ); + + ASSERT_EQUAL( object.find_field("a").get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( object.find_field("b").get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( object.find_field("c/d").get_uint64().value_unsafe(), 3 ); + + ASSERT_ERROR( object.find_field("a"), NO_SUCH_FIELD ); + ASSERT_ERROR( object.find_field("d"), NO_SUCH_FIELD ); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + simdjson_result object; + object = doc_result.get_object(); + + ASSERT_EQUAL( object.find_field("a").get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( object.find_field("b").get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( object.find_field("c/d").get_uint64().value_unsafe(), 3 ); + + ASSERT_ERROR( object.find_field("a"), NO_SUCH_FIELD ); + ASSERT_ERROR( object.find_field("d"), NO_SUCH_FIELD ); + return true; + })); + TEST_SUCCEED(); + } + + bool document_object_find_field() { + TEST_START(); + auto json = R"({ "a": 1, "b": 2, "c/d": 3})"_padded; + SUBTEST("singlestage::document", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::document doc; + ASSERT_SUCCESS( std::move(doc_result).get(doc) ); + ASSERT_EQUAL( doc.find_field("a").get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( doc.find_field("b").get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( doc.find_field("c/d").get_uint64().value_unsafe(), 3 ); + + ASSERT_ERROR( doc.find_field("a"), NO_SUCH_FIELD ); + ASSERT_ERROR( doc.find_field("d"), NO_SUCH_FIELD ); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + ASSERT_EQUAL( doc_result.find_field("a").get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( doc_result.find_field("b").get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( doc_result.find_field("c/d").get_uint64().value_unsafe(), 3 ); + + ASSERT_ERROR( doc_result.find_field("a"), NO_SUCH_FIELD ); + ASSERT_ERROR( doc_result.find_field("d"), NO_SUCH_FIELD ); + return true; + })); + TEST_SUCCEED(); + } + + bool value_object_find_field() { + TEST_START(); + auto json = R"({ "outer": { "a": 1, "b": 2, "c/d": 3 } })"_padded; + SUBTEST("singlestage::value", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::value object; + ASSERT_SUCCESS( doc_result.find_field("outer").get(object) ); + ASSERT_EQUAL( object.find_field("a").get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( object.find_field("b").get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( object.find_field("c/d").get_uint64().value_unsafe(), 3 ); + + ASSERT_ERROR( object.find_field("a"), NO_SUCH_FIELD ); + ASSERT_ERROR( object.find_field("d"), NO_SUCH_FIELD ); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + simdjson_result object = doc_result.find_field("outer"); + ASSERT_EQUAL( object.find_field("a").get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( object.find_field("b").get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( object.find_field("c/d").get_uint64().value_unsafe(), 3 ); + + ASSERT_ERROR( object.find_field("a"), NO_SUCH_FIELD ); + ASSERT_ERROR( object.find_field("d"), NO_SUCH_FIELD ); + return true; + })); + TEST_SUCCEED(); + } + + bool run() { + return + object_find_field_unordered() && + document_object_find_field_unordered() && + value_object_find_field_unordered() && + object_find_field() && + document_object_find_field() && + value_object_find_field() && + true; + } + +} // namespace object_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, object_tests::run); +} diff --git a/tests/singlestage/singlestage_object_index_tests.cpp b/tests/singlestage/singlestage_object_index_tests.cpp new file mode 100644 index 000000000..c4ded7e77 --- /dev/null +++ b/tests/singlestage/singlestage_object_index_tests.cpp @@ -0,0 +1,432 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace object_tests { + using namespace std; + using simdjson::singlestage::json_type; + + bool object_index() { + TEST_START(); + auto json = R"({ "a": 1, "b": 2, "c/d": 3})"_padded; + SUBTEST("singlestage::object", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_SUCCESS( doc_result.get(object) ); + + ASSERT_EQUAL( object["a"].get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( object["b"].get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( object["c/d"].get_uint64().value_unsafe(), 3 ); + + ASSERT_EQUAL( object["a"].get_uint64().value_unsafe(), 1 ); + ASSERT_ERROR( object["d"], NO_SUCH_FIELD ); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + simdjson_result object; + object = doc_result.get_object(); + + ASSERT_EQUAL( object["a"].get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( object["b"].get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( object["c/d"].get_uint64().value_unsafe(), 3 ); + + ASSERT_EQUAL( object["a"].get_uint64().value_unsafe(), 1 ); + ASSERT_ERROR( object["d"], NO_SUCH_FIELD ); + return true; + })); + TEST_SUCCEED(); + } + + bool document_object_index() { + TEST_START(); + auto json = R"({ "a": 1, "b": 2, "c/d": 3})"_padded; + SUBTEST("singlestage::document", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::document doc; + ASSERT_SUCCESS( std::move(doc_result).get(doc) ); + ASSERT_EQUAL( doc["a"].get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( doc["b"].get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( doc["c/d"].get_uint64().value_unsafe(), 3 ); + + ASSERT_EQUAL( doc["a"].get_uint64().value_unsafe(), 1 ); + ASSERT_ERROR( doc["d"], NO_SUCH_FIELD ); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + ASSERT_EQUAL( doc_result["a"].get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( doc_result["b"].get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( doc_result["c/d"].get_uint64().value_unsafe(), 3 ); + + ASSERT_EQUAL( doc_result["a"].get_uint64().value_unsafe(), 1 ); + ASSERT_ERROR( doc_result["d"], NO_SUCH_FIELD ); + return true; + })); + TEST_SUCCEED(); + } + + bool value_object_index() { + TEST_START(); + auto json = R"({ "outer": { "a": 1, "b": 2, "c/d": 3 } })"_padded; + SUBTEST("singlestage::value", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::value object; + ASSERT_SUCCESS( doc_result["outer"].get(object) ); + ASSERT_EQUAL( object["a"].get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( object["b"].get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( object["c/d"].get_uint64().value_unsafe(), 3 ); + + ASSERT_EQUAL( object["a"].get_uint64().value_unsafe(), 1 ); + ASSERT_ERROR( object["d"], NO_SUCH_FIELD ); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + simdjson_result object = doc_result["outer"]; + ASSERT_EQUAL( object["a"].get_uint64().value_unsafe(), 1 ); + ASSERT_EQUAL( object["b"].get_uint64().value_unsafe(), 2 ); + ASSERT_EQUAL( object["c/d"].get_uint64().value_unsafe(), 3 ); + + ASSERT_EQUAL( object["a"].get_uint64().value_unsafe(), 1 ); + ASSERT_ERROR( object["d"], NO_SUCH_FIELD ); + return true; + })); + TEST_SUCCEED(); + } + + bool document_nested_object_index() { + TEST_START(); + auto json = R"({ "x": { "y": { "z": 2 } } }})"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + ASSERT_EQUAL( doc_result["x"]["y"]["z"].get_uint64().value_unsafe(), 2 ); + return true; + })); + SUBTEST("singlestage::document", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::document doc; + ASSERT_SUCCESS( std::move(doc_result).get(doc) ); + ASSERT_EQUAL( doc["x"]["y"]["z"].get_uint64().value_unsafe(), 2 ); + return true; + })); + TEST_SUCCEED(); + } + + bool nested_object_index() { + TEST_START(); + auto json = R"({ "x": { "y": { "z": 2 } } }})"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + simdjson_result object = doc_result.get_object(); + ASSERT_EQUAL( object["x"]["y"]["z"].get_uint64().value_unsafe(), 2 ); + return true; + })); + SUBTEST("singlestage::object", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_SUCCESS( doc_result.get(object) ); + ASSERT_EQUAL( object["x"]["y"]["z"].get_uint64().value_unsafe(), 2 ); + return true; + })); + TEST_SUCCEED(); + } + + bool value_nested_object_index() { + TEST_START(); + auto json = R"({ "x": { "y": { "z": 2 } } }})"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + simdjson_result x = doc_result["x"]; + ASSERT_EQUAL( x["y"]["z"].get_uint64().value_unsafe(), 2 ); + return true; + })); + SUBTEST("singlestage::value", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::value x; + ASSERT_SUCCESS( doc_result["x"].get(x) ); + ASSERT_EQUAL( x["y"]["z"].get_uint64().value_unsafe(), 2 ); + return true; + })); + TEST_SUCCEED(); + } + + bool object_index_partial_children() { + TEST_START(); + auto json = R"( + { + "scalar_ignore": 0, + "empty_array_ignore": [], + "empty_object_ignore": {}, + "object_break": { "x": 3, "y": 33 }, + "object_break_unused": { "x": 4, "y": 44 }, + "object_index": { "x": 5, "y": 55 }, + "object_index_unused": { "x": 6, "y": 66 }, + "array_break": [ 7, 77, 777 ], + "array_break_unused": [ 8, 88, 888 ], + "quadruple_nested_break": { "a": [ { "b": [ 9, 99 ], "c": 999 }, 9999 ], "d": 99999 }, + "actual_value": 10 + } + )"_padded; + SUBTEST("singlestage::object", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_SUCCESS( doc_result.get(object) ); + + ASSERT_SUCCESS( object["scalar_ignore"] ); + std::cout << " - After ignoring empty scalar ..." << std::endl; + + ASSERT_SUCCESS( object["empty_array_ignore"] ); + std::cout << " - After ignoring empty array ..." << std::endl; + + ASSERT_SUCCESS( object["empty_object_ignore"] ); + std::cout << " - After ignoring empty object ..." << std::endl; + + // Break after using first value in child object + { + auto value = object["object_break"]; + for (auto [ child_field, error ] : value.get_object()) { + ASSERT_SUCCESS(error); + ASSERT_EQUAL(child_field.key(), "x"); + uint64_t x; + ASSERT_SUCCESS( child_field.value().get(x) ); + ASSERT_EQUAL(x, 3); + break; // Break after the first value + } + std::cout << " - After using first value in child object ..." << std::endl; + } + + // Break without using first value in child object + { + auto value = object["object_break_unused"]; + for (auto [ child_field, error ] : value.get_object()) { + ASSERT_SUCCESS(error); + ASSERT_EQUAL(child_field.key(), "x"); + break; + } + std::cout << " - After reaching (but not using) first value in child object ..." << std::endl; + } + + // Only look up one field in child object + { + auto value = object["object_index"]; + + uint64_t x; + ASSERT_SUCCESS( value["x"].get(x) ); + ASSERT_EQUAL( x, 5 ); + std::cout << " - After looking up one field in child object ..." << std::endl; + } + + // Only look up one field in child object, but don't use it + { + auto value = object["object_index_unused"]; + + ASSERT_SUCCESS( value["x"] ); + std::cout << " - After looking up (but not using) one field in child object ..." << std::endl; + } + + // Break after first value in child array + { + auto value = object["array_break"]; + + for (auto child_value : value) { + uint64_t x; + ASSERT_SUCCESS( child_value.get(x) ); + ASSERT_EQUAL( x, 7 ); + break; + } + std::cout << " - After using first value in child array ..." << std::endl; + } + + // Break without using first value in child array + { + auto value = object["array_break_unused"]; + + for (auto child_value : value) { + ASSERT_SUCCESS(child_value); + break; + } + std::cout << " - After reaching (but not using) first value in child array ..." << std::endl; + } + + // Break out of multiple child loops + { + auto value = object["quadruple_nested_break"]; + for (auto child1 : value.get_object()) { + for (auto child2 : child1.value().get_array()) { + for (auto child3 : child2.get_object()) { + for (auto child4 : child3.value().get_array()) { + uint64_t x; + ASSERT_SUCCESS( child4.get(x) ); + ASSERT_EQUAL( x, 9 ); + break; + } + break; + } + break; + } + break; + } + std::cout << " - After breaking out of quadruply-nested arrays and objects ..." << std::endl; + } + + // Test the actual value + { + auto value = object["actual_value"]; + uint64_t actual_value; + ASSERT_SUCCESS( value.get(actual_value) ); + ASSERT_EQUAL( actual_value, 10 ); + } + + return true; + })); + + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + ASSERT_SUCCESS( doc_result["scalar_ignore"] ); + std::cout << " - After ignoring empty scalar ..." << std::endl; + + ASSERT_SUCCESS( doc_result["empty_array_ignore"] ); + std::cout << " - After ignoring empty array ..." << std::endl; + + ASSERT_SUCCESS( doc_result["empty_object_ignore"] ); + std::cout << " - After ignoring empty doc_result ..." << std::endl; + + // Break after using first value in child object + { + auto value = doc_result["object_break"]; + for (auto [ child_field, error ] : value.get_object()) { + ASSERT_SUCCESS(error); + ASSERT_EQUAL(child_field.key(), "x"); + uint64_t x; + ASSERT_SUCCESS( child_field.value().get(x) ); + ASSERT_EQUAL(x, 3); + break; // Break after the first value + } + std::cout << " - After using first value in child object ..." << std::endl; + } + + // Break without using first value in child object + { + auto value = doc_result["object_break_unused"]; + for (auto [ child_field, error ] : value.get_object()) { + ASSERT_SUCCESS(error); + ASSERT_EQUAL(child_field.key(), "x"); + break; + } + std::cout << " - After reaching (but not using) first value in child object ..." << std::endl; + } + + // Only look up one field in child object + { + auto value = doc_result["object_index"]; + + uint64_t x; + ASSERT_SUCCESS( value["x"].get(x) ); + ASSERT_EQUAL( x, 5 ); + std::cout << " - After looking up one field in child object ..." << std::endl; + } + + // Only look up one field in child object, but don't use it + { + auto value = doc_result["object_index_unused"]; + + ASSERT_SUCCESS( value["x"] ); + std::cout << " - After looking up (but not using) one field in child object ..." << std::endl; + } + + // Break after first value in child array + { + auto value = doc_result["array_break"]; + + for (auto child_value : value) { + uint64_t x; + ASSERT_SUCCESS( child_value.get(x) ); + ASSERT_EQUAL( x, 7 ); + break; + } + std::cout << " - After using first value in child array ..." << std::endl; + } + + // Break without using first value in child array + { + auto value = doc_result["array_break_unused"]; + + for (auto child_value : value) { + ASSERT_SUCCESS(child_value); + break; + } + std::cout << " - After reaching (but not using) first value in child array ..." << std::endl; + } + + // Break out of multiple child loops + { + auto value = doc_result["quadruple_nested_break"]; + for (auto child1 : value.get_object()) { + for (auto child2 : child1.value().get_array()) { + for (auto child3 : child2.get_object()) { + for (auto child4 : child3.value().get_array()) { + uint64_t x; + ASSERT_SUCCESS( child4.get(x) ); + ASSERT_EQUAL( x, 9 ); + break; + } + break; + } + break; + } + break; + } + std::cout << " - After breaking out of quadruply-nested arrays and objects ..." << std::endl; + } + + // Test the actual value + { + auto value = doc_result["actual_value"]; + uint64_t actual_value; + ASSERT_SUCCESS( value.get(actual_value) ); + ASSERT_EQUAL( actual_value, 10 ); + } + + return true; + })); + + return true; + } + +#if SIMDJSON_EXCEPTIONS + + bool object_index_exception() { + TEST_START(); + auto json = R"({ "a": 1, "b": 2, "c/d": 3})"_padded; + SUBTEST("singlestage::object", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object = doc_result; + + ASSERT_EQUAL( uint64_t(object["a"]), 1 ); + ASSERT_EQUAL( uint64_t(object["b"]), 2 ); + ASSERT_EQUAL( uint64_t(object["c/d"]), 3 ); + + return true; + })); + TEST_SUCCEED(); + } + bool nested_object_index_exception() { + TEST_START(); + auto json = R"({ "x": { "y": { "z": 2 } } }})"_padded; + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + ASSERT_EQUAL( uint64_t(doc_result["x"]["y"]["z"]), 2 ); + return true; + })); + TEST_SUCCEED(); + } + +#endif // SIMDJSON_EXCEPTIONS + + bool run() { + return + object_index() && + document_object_index() && + value_object_index() && + nested_object_index() && + document_nested_object_index() && + value_nested_object_index() && + object_index_partial_children() && +#if SIMDJSON_EXCEPTIONS + object_index_exception() && + nested_object_index_exception() && +#endif // SIMDJSON_EXCEPTIONS + true; + } + +} // namespace object_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, object_tests::run); +} diff --git a/tests/singlestage/singlestage_object_tests.cpp b/tests/singlestage/singlestage_object_tests.cpp new file mode 100644 index 000000000..a3d221d33 --- /dev/null +++ b/tests/singlestage/singlestage_object_tests.cpp @@ -0,0 +1,1344 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace object_tests { + using namespace std; + using simdjson::singlestage::json_type; + + bool issue1979() { + TEST_START(); + auto json = R"({ + "@avito-core/toggles:6.1.18": { + "add_model_review_from": true + } + })"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::object object; + ASSERT_SUCCESS(doc.get_object().get(object)); + ASSERT_SUCCESS(object["@avito-core/toggles:6.1.18"].get_object().error()); + TEST_SUCCEED(); + } + + bool issue1977() { + TEST_START(); + auto json = R"({"1": 2} foo })"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::object object; + ASSERT_SUCCESS(doc.get_object().get(object)); + for (auto values : object) { + ASSERT_SUCCESS(values); + } + ASSERT_FALSE(doc.at_end()); + TEST_SUCCEED(); + } + + bool issue1745() { + TEST_START(); + auto json = R"({ + "keys": [ + { + "kid": "bbd2ac7c4c5eb8adc8eeffbc8f5a2dd6cf7545e4", + "e": "AQAB", + "alg": "RS256", + "use": "sig", + "n": "y930dtGTeMG52IPsKmMuEpPHLaxuYQlduZd6BqFVjc2-UFZR8fNqtnYzAjbXWJD_Tqxgdlj_MW4vogvX4sHwVpZONvdyeGoIyDQtis6iuGQhQamV85F_JbrEUnEw3QCO87Liz5UXG6BK2HRyPhDfMex1_tO0ROmySLFdCTS17D0wah71Ibpi0gI8LUi6kzVRjYDIC1oE-iK3Y9s88Bi4ZGYJxXAbnNwbwVkGOKCXja9k0jjBGRxZD-4KDuf493lFOOEGSLDA2Qp9rDqrURP12XYgvf_zJx_kSDipnr0gL6Vz2n3H4-XN4tA45zuzRkHoE7-XexPq-tv7kQ8pSjY2uQ", + "kty": "RSA" + }, + { + "kty": "RSA", + "n": "zMHxWuxztMKXdBhv3rImlUvW_yp6nO03cVXPyA0Vyq0-M7LfOJJIF-OdNoRGdsFPHVKCFoo6qGhR8rBCmMxA4fM-Ubk5qKuUqCN9eP3yZJq8Cw9tUrt_qh7uW-qfMr0upcyeSHhC_zW1lTGs5sowDorKN_jQ1Sfh9hfBxfc8T7dQAAgEqqMcE3u-2J701jyhJz0pvurCfziiB3buY6SGREhBQwNwpnQjt_lE2U4km8FS0woPzt0ccE3zsGL2qM-LWZbOm9aXquSnqNJLt3tGVvShnev-GiJ1XfQ3EWm0f4w0TX9fTOkxstl0vo_vW_FjGQ0D1pXSjqb7n-hAdXwc9w", + "alg": "RS256", + "kid": "85828c59284a69b54b27483e487c3bd46cd2a2b3", + "e": "AQAB", + "use": "sig" + } + ] +} + + })"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + for (auto items_instance : doc["keys"]) { + singlestage::object items; + ASSERT_SUCCESS(items_instance.get_object().get(items)); + std::string_view kty_value; + ASSERT_SUCCESS(items["kty"].get_string().get(kty_value)); + std::string_view use_value; + ASSERT_SUCCESS(items["use"].get_string().get(use_value)); + if (kty_value != "RSA" && use_value != "sig") { continue; } + std::string_view str; + ASSERT_SUCCESS(items["alg"].get_string().get(str)); + ASSERT_EQUAL(str, "RS256"); + ASSERT_EQUAL(use_value, "sig"); + ASSERT_EQUAL(kty_value, "RSA"); + ASSERT_SUCCESS(items["n"].get_string().get(str)); + ASSERT_EQUAL(str.size(), 342); + ASSERT_SUCCESS(items["e"].get_string().get(str)); + ASSERT_EQUAL(str, "AQAB"); + } + TEST_SUCCEED(); + } + + bool issue1723() { + TEST_START(); + simdjson::singlestage::parser parser; + simdjson::padded_string docdata = R"({ + "contents": + { + "bids":[ + {"offset":"1", "price":"1.1", "size":"4.1"} + ], + "asks":[ + {"offset":"1111", "price":"1.100", "size":"114.1"} + ] + + } + })"_padded; + simdjson::singlestage::document doc; + auto error = parser.iterate(docdata).get(doc); + if(error) { return false; } + singlestage::object content; + error = doc.get_object().find_field("contents").get(content); + if(error) { return false; } + singlestage::array bids; + error = content["bids"].get_array().get(bids); + if(error) { return false; } + for (auto aux : bids) { + uint64_t id; + error = aux["offset"].get_uint64_in_string().get(id); + if(error) { return false; } + double price; + error = aux["price"].get_double_in_string().get(price); + if(error) { return false; } + double size; + error = aux["size"].get_double_in_string().get(size); + if(error) { return false; } + std::cout << id << " " << price << " " << size << std::endl; + } + singlestage::array asks; + error = content["asks"].get_array().get(asks); + if(error) { return false; } + for (auto aux : asks) { + uint64_t id; + error = aux["offset"].get_uint64_in_string().get(id); + if(error) { return false; } + double price; + error = aux["price"].get_double_in_string().get(price); + if(error) { return false; } + double size; + error = aux["size"].get_double_in_string().get(size); + if(error) { return false; } + std::cout << id << " " << price << " " << size << std::endl; + } + return true; + } + + // In this test, no non-trivial object in an array have a missing key + bool no_missing_keys() { + TEST_START(); + simdjson::singlestage::parser parser; + simdjson::padded_string docdata = R"([{"a":"a"},{}])"_padded; + simdjson::singlestage::document doc; + auto error = parser.iterate(docdata).get(doc); + if(error != simdjson::SUCCESS) { return false; } + simdjson::singlestage::array a; + error = doc.get_array().get(a); + if(error != simdjson::SUCCESS) { return false; } + size_t counter{0}; + for(auto elem : a) { + error = elem.find_field_unordered("a").error(); + if(counter == 0) { + ASSERT_EQUAL( error, simdjson::SUCCESS); + } else { + ASSERT_EQUAL( error, simdjson::NO_SUCH_FIELD); + } + counter++; + } + return true; + } + + bool missing_key_continue() { + TEST_START(); + simdjson::singlestage::parser parser; + simdjson::padded_string docdata = R"({"a":0, "b":1, "c":2})"_padded; + simdjson::singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + int64_t num; + ASSERT_SUCCESS(doc["a"].get(num)); + ASSERT_EQUAL(num, 0); + ASSERT_SUCCESS(doc["b"].get(num)); + ASSERT_EQUAL(num, 1); + ASSERT_SUCCESS(doc["c"].get(num)); + ASSERT_EQUAL(num, 2); + // Start again, but omit a key + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + ASSERT_SUCCESS(doc["a"].get(num)); + ASSERT_EQUAL(num, 0); + ASSERT_SUCCESS(doc["c"].get(num)); + ASSERT_EQUAL(num, 2); + // Start again, but request a missing key + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + simdjson::singlestage::object obj; + ASSERT_SUCCESS(doc.get_object().get(obj)); + ASSERT_SUCCESS(obj["a"].get(num)); + ASSERT_EQUAL(num, 0); + assert_error(obj["d"].get(num), NO_SUCH_FIELD); + ASSERT_SUCCESS(obj["c"].get(num)); + ASSERT_EQUAL(num, 2); + // Start again, but request a missing key first + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + ASSERT_SUCCESS(doc.get_object().get(obj)); + assert_error(obj["d"].get(num), NO_SUCH_FIELD); + ASSERT_SUCCESS(obj["a"].get(num)); + ASSERT_EQUAL(num, 0); + // Start again, but request a missing key twice + ASSERT_SUCCESS(parser.iterate(docdata).get(doc)); + ASSERT_SUCCESS(doc.get_object().get(obj)); + ASSERT_SUCCESS(obj["a"].get(num)); + ASSERT_EQUAL(num, 0); + assert_error(obj["d"].get(num), NO_SUCH_FIELD); + assert_error(obj["z"].get(num), NO_SUCH_FIELD); + // Because we do a full circle, you can query the same + // key twice!!! + ASSERT_SUCCESS(obj["a"].get(num)); + ASSERT_EQUAL(num, 0); + ASSERT_SUCCESS(obj["b"].get(num)); + ASSERT_EQUAL(num, 1); + ASSERT_SUCCESS(obj["b"].get(num)); + ASSERT_EQUAL(num, 1); + ASSERT_SUCCESS(obj["a"].get(num)); + ASSERT_EQUAL(num, 0); + assert_error(obj["d"].get(num), NO_SUCH_FIELD); + ASSERT_SUCCESS(obj["a"].get(num)); + ASSERT_EQUAL(num, 0); + TEST_SUCCEED(); + } + + bool missing_keys() { + TEST_START(); + simdjson::singlestage::parser parser; + simdjson::padded_string docdata = R"([{"a":"a"},{}])"_padded; + simdjson::singlestage::document doc; + auto error = parser.iterate(docdata).get(doc); + if(error != simdjson::SUCCESS) { return false; } + simdjson::singlestage::array a; + error = doc.get_array().get(a); + if(error != simdjson::SUCCESS) { return false; } + for(auto elem : a) { + error = elem.find_field_unordered("keynotfound").error(); + if(error != simdjson::NO_SUCH_FIELD) { + std::cout << error << std::endl; + return false; + } + } + return true; + } + + bool missing_keys_for_empty_top_level_object() { + TEST_START(); + simdjson::singlestage::parser parser; + simdjson::padded_string docdata = "{}"_padded; + simdjson::singlestage::document doc; + auto error = parser.iterate(docdata).get(doc); + if(error != simdjson::SUCCESS) { return false; } + error = doc.find_field_unordered("keynotfound").error(); + if(error != simdjson::NO_SUCH_FIELD) { + std::cout << error << std::endl; + return false; + } + return true; + } + +#if SIMDJSON_EXCEPTIONS + + bool issue1965() { + TEST_START(); + std::string str = "{\"query\":\"ah\"}"; + std::unique_ptr buffer(new char[str.size() + simdjson::SIMDJSON_PADDING]); + memcpy(buffer.get(), str.data(), str.size()); + simdjson::padded_string_view view(buffer.get(), str.size(), str.size() + simdjson::SIMDJSON_PADDING); + simdjson::singlestage::parser parser; + simdjson::singlestage::document doc = parser.iterate(view); + simdjson::singlestage::object root = doc.get_object(); + simdjson::singlestage::value query = root.find_field("query"); + simdjson::singlestage::raw_json_string raw = query.get_raw_json_string(); + std::unique_ptr dst_buffer(new uint8_t[3 + simdjson::SIMDJSON_PADDING]); + uint8_t * dst = dst_buffer.get(); + std::string_view fieldstring = parser.unescape(raw, dst); + std::cout << fieldstring << std::endl; + TEST_SUCCEED(); + } + + bool issue1745_with_exceptions() { + TEST_START(); + auto json = R"({ + "keys": [ + { + "kid": "bbd2ac7c4c5eb8adc8eeffbc8f5a2dd6cf7545e4", + "e": "AQAB", + "alg": "RS256", + "use": "sig", + "n": "y930dtGTeMG52IPsKmMuEpPHLaxuYQlduZd6BqFVjc2-UFZR8fNqtnYzAjbXWJD_Tqxgdlj_MW4vogvX4sHwVpZONvdyeGoIyDQtis6iuGQhQamV85F_JbrEUnEw3QCO87Liz5UXG6BK2HRyPhDfMex1_tO0ROmySLFdCTS17D0wah71Ibpi0gI8LUi6kzVRjYDIC1oE-iK3Y9s88Bi4ZGYJxXAbnNwbwVkGOKCXja9k0jjBGRxZD-4KDuf493lFOOEGSLDA2Qp9rDqrURP12XYgvf_zJx_kSDipnr0gL6Vz2n3H4-XN4tA45zuzRkHoE7-XexPq-tv7kQ8pSjY2uQ", + "kty": "RSA" + }, + { + "kty": "RSA", + "n": "zMHxWuxztMKXdBhv3rImlUvW_yp6nO03cVXPyA0Vyq0-M7LfOJJIF-OdNoRGdsFPHVKCFoo6qGhR8rBCmMxA4fM-Ubk5qKuUqCN9eP3yZJq8Cw9tUrt_qh7uW-qfMr0upcyeSHhC_zW1lTGs5sowDorKN_jQ1Sfh9hfBxfc8T7dQAAgEqqMcE3u-2J701jyhJz0pvurCfziiB3buY6SGREhBQwNwpnQjt_lE2U4km8FS0woPzt0ccE3zsGL2qM-LWZbOm9aXquSnqNJLt3tGVvShnev-GiJ1XfQ3EWm0f4w0TX9fTOkxstl0vo_vW_FjGQ0D1pXSjqb7n-hAdXwc9w", + "alg": "RS256", + "kid": "85828c59284a69b54b27483e487c3bd46cd2a2b3", + "e": "AQAB", + "use": "sig" + } + ] +} + + })"_padded; + singlestage::parser parser; + singlestage::document doc = parser.iterate(json); + for (auto items : doc["keys"]) { + std::string_view kty_value = items["kty"].get_string(); + std::string_view use_value = items["use"].get_string(); + if (kty_value != "RSA" && use_value != "sig") { continue; } + std::string_view str = items["alg"].get_string(); + if(str != "RS256") { return false; } + if(use_value != "sig") { return false; } + if(kty_value != "RSA") { return false; } + str = items["n"].get_string(); + if(str.size() != 342) { return false; } + str = items["e"].get_string(); + if(str != "AQAB") { return false; } + } + TEST_SUCCEED(); + } + + bool issue1723_except() { + TEST_START(); + simdjson::singlestage::parser parser; + simdjson::padded_string docdata = R"({ + "contents": + { + "bids":[ + {"offset":"1", "price":"1.1", "size":"4.1"} + ], + "asks":[ + {"offset":"1111", "price":"1.100", "size":"114.1"} + ] + + } + })"_padded; + simdjson::singlestage::document doc = parser.iterate(docdata); + singlestage::object content = doc.get_object().find_field("contents"); + singlestage::array bids = content["bids"].get_array(); + std::cout << "bids:" << std::endl; + for (auto aux : bids) { + uint64_t id = aux["offset"].get_uint64_in_string(); + double price = aux["price"].get_double_in_string(); + double size = aux["size"].get_double_in_string(); + std::cout << id << " " << price << " " << size << std::endl; + } + singlestage::array asks = content["asks"].get_array(); + std::cout << "asks:" << std::endl; + for (auto aux : asks) { + uint64_t id = aux["offset"].get_uint64_in_string(); + double price = aux["price"].get_double_in_string(); + double size = aux["size"].get_double_in_string(); + std::cout << id << " " << price << " " << size << std::endl; + } + return true; + } + // used in issue_1521 + // difficult to use as a lambda because it is recursive. + void broken_descend(singlestage::object node) { + if(auto type = node.find_field_unordered("type"); type.error() == SUCCESS && type == "child") { + auto n = node.find_field_unordered("name"); + if(n.error() == simdjson::SUCCESS) { + std::cout << std::string_view(n) << std::endl; + } + } else { + for (singlestage::object child_node : node["nodes"]) { broken_descend(child_node); } + } + } + + bool broken_issue_1521() { + TEST_START(); + singlestage::parser parser; + padded_string json = R"({"type":"root","nodes":[{"type":"child","nodes":[]},{"type":"child","name":"child-name","nodes":[]}]})"_padded; + singlestage::document file_tree = parser.iterate(json); + try { + broken_descend(file_tree); + } catch(simdjson::simdjson_error& e) { + std::cout << "The document is valid JSON: " << json << std::endl; + TEST_FAIL(e.error()); + } + TEST_SUCCEED(); + } + + bool fixed_broken_issue_1521() { + TEST_START(); + singlestage::parser parser; + // We omit the ',"nodes":[]' + padded_string json = R"({"type":"root","nodes":[{"type":"child"},{"type":"child","name":"child-name","nodes":[]}]})"_padded; + singlestage::document file_tree = parser.iterate(json); + try { + broken_descend(file_tree); + } catch(simdjson::simdjson_error& e) { + std::cout << "The document is valid JSON: " << json << std::endl; + TEST_FAIL(e.error()); + } + TEST_SUCCEED(); + } + + // used in issue_1521 + // difficult to use as a lambda because it is recursive. + void descend(singlestage::object node) { + auto n = node.find_field_unordered("name"); + if(auto type = node.find_field_unordered("type"); type.error() == SUCCESS && type == "child") { + if(n.error() == simdjson::SUCCESS) { + std::cout << std::string_view(n) << std::endl; + } + } else { + for (singlestage::object child_node : node["nodes"]) { descend(child_node); } + } + } + + bool issue_1521() { + TEST_START(); + singlestage::parser parser; + padded_string json = R"({"type":"root","nodes":[{"type":"child","nodes":[]},{"type":"child","name":"child-name","nodes":[]}]})"_padded; + singlestage::document file_tree = parser.iterate(json); + try { + descend(file_tree); + } catch(simdjson::simdjson_error& e) { + std::cout << "The document is valid JSON: " << json << std::endl; + TEST_FAIL(e.error()); + } + TEST_SUCCEED(); + } +#endif + + bool iterate_object() { + TEST_START(); + auto json = R"({ "a": 1, "b": 2, "c": 3 })"_padded; + const char* expected_key[] = { "a", "b", "c" }; + const uint64_t expected_value[] = { 1, 2, 3 }; + SUBTEST("singlestage::object", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_RESULT( doc_result.type(), json_type::object ); + ASSERT_SUCCESS( doc_result.get(object) ); + size_t i = 0; + for (auto field : object) { + ASSERT_SUCCESS( field.key().error() ); + ASSERT_EQUAL( field.key().value_unsafe(), expected_key[i]); + ASSERT_EQUAL( field.value().get_uint64().value_unsafe(), expected_value[i] ); + i++; + } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); + return true; + })); + SUBTEST("singlestage::object-document-rewind", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_RESULT( doc_result.type(), json_type::object ); + ASSERT_SUCCESS( doc_result.get(object) ); + size_t i = 0; + for (auto field : object) { + ASSERT_SUCCESS( field.error() ); + ASSERT_EQUAL( field.key().value_unsafe(), expected_key[i]); + ASSERT_EQUAL( field.value().get_uint64().value_unsafe(), expected_value[i] ); + i++; + } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); + doc_result.rewind(); + ASSERT_RESULT( doc_result.type(), json_type::object ); + ASSERT_SUCCESS( doc_result.get(object) ); + i = 0; + for (auto field : object) { + ASSERT_SUCCESS( field.error() ); + ASSERT_EQUAL( field.key().value_unsafe(), expected_key[i]); + ASSERT_EQUAL( field.value().get_uint64().value_unsafe(), expected_value[i] ); + i++; + } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); + return true; + })); + SUBTEST("singlestage::object-rewind", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_RESULT( doc_result.type(), json_type::object ); + ASSERT_SUCCESS( doc_result.get(object) ); + size_t i = 0; + for (auto field : object) { + ASSERT_SUCCESS( field.error() ); + ASSERT_EQUAL( field.key().value_unsafe(), expected_key[i]); + ASSERT_EQUAL( field.value().get_uint64().value_unsafe(), expected_value[i] ); + i++; + } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); + object.reset(); + i = 0; + for (auto field : object) { + ASSERT_SUCCESS( field.error() ); + ASSERT_EQUAL( field.key().value_unsafe(), expected_key[i]); + ASSERT_EQUAL( field.value().get_uint64().value_unsafe(), expected_value[i] ); + i++; + } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + simdjson_result object_result = doc_result.get_object(); + size_t i = 0; + for (auto field : object_result) { + ASSERT_SUCCESS( field.error() ); + ASSERT_EQUAL( field.key().value_unsafe(), expected_key[i]); + ASSERT_EQUAL( field.value().get_uint64().value_unsafe(), expected_value[i] ); + i++; + } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); + return true; + })); + TEST_SUCCEED(); + } + + bool iterate_object_partial_children() { + TEST_START(); + auto json = R"( + { + "scalar_ignore": 0, + "empty_array_ignore": [], + "empty_object_ignore": {}, + "object_break": { "x": 3, "y": 33 }, + "object_break_unused": { "x": 4, "y": 44 }, + "object_index": { "x": 5, "y": 55 }, + "object_index_unused": { "x": 6, "y": 66 }, + "array_break": [ 7, 77, 777 ], + "array_break_unused": [ 8, 88, 888 ], + "quadruple_nested_break": { "a": [ { "b": [ 9, 99 ], "c": 999 }, 9999 ], "d": 99999 }, + "actual_value": 10 + } + )"_padded; + SUBTEST("singlestage::object", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_SUCCESS( doc_result.get(object) ); + size_t i = 0; + for (auto field : object) { + singlestage::raw_json_string key; + ASSERT_SUCCESS( field.key().get(key) ); + + switch (i) { + case 0: { + ASSERT_EQUAL(key, "scalar_ignore"); + std::cout << " - After ignoring empty scalar ..." << std::endl; + break; + } + case 1: { + ASSERT_EQUAL(key, "empty_array_ignore"); + std::cout << " - After ignoring empty array ..." << std::endl; + break; + } + case 2: { + ASSERT_EQUAL(key, "empty_object_ignore"); + std::cout << " - After ignoring empty object ..." << std::endl; + break; + } + // Break after using first value in child object + case 3: { + ASSERT_EQUAL(key, "object_break"); + + for (auto child_field : field.value().get_object()) { + ASSERT_SUCCESS( child_field.error() ); + ASSERT_EQUAL(child_field.key().value_unsafe(), "x"); + uint64_t x; + ASSERT_SUCCESS( child_field.value().get(x) ); + ASSERT_EQUAL(x, 3); + break; // Break after the first value + } + std::cout << " - After using first value in child object ..." << std::endl; + break; + } + + // Break without using first value in child object + case 4: { + ASSERT_EQUAL(key, "object_break_unused"); + + for (auto child_field : field.value().get_object()) { + ASSERT_SUCCESS(child_field.error()); + ASSERT_EQUAL(child_field.key().value_unsafe(), "x"); + break; + } + std::cout << " - After reaching (but not using) first value in child object ..." << std::endl; + break; + } + + // Only look up one field in child object + case 5: { + ASSERT_EQUAL(key, "object_index"); + + uint64_t x; + ASSERT_SUCCESS( field.value()["x"].get(x) ); + ASSERT_EQUAL( x, 5 ); + std::cout << " - After looking up one field in child object ..." << std::endl; + break; + } + + // Only look up one field in child object, but don't use it + case 6: { + ASSERT_EQUAL(key, "object_index_unused"); + + ASSERT_SUCCESS( field.value()["x"] ); + std::cout << " - After looking up (but not using) one field in child object ..." << std::endl; + break; + } + + // Break after first value in child array + case 7: { + ASSERT_EQUAL(key, "array_break"); + for (auto child_value : field.value()) { + uint64_t x; + ASSERT_SUCCESS( child_value.get(x) ); + ASSERT_EQUAL( x, 7 ); + break; + } + std::cout << " - After using first value in child array ..." << std::endl; + break; + } + + // Break without using first value in child array + case 8: { + ASSERT_EQUAL(key, "array_break_unused"); + for (auto child_value : field.value()) { + ASSERT_SUCCESS(child_value); + break; + } + std::cout << " - After reaching (but not using) first value in child array ..." << std::endl; + break; + } + + // Break out of multiple child loops + case 9: { + ASSERT_EQUAL(key, "quadruple_nested_break"); + for (auto child1 : field.value().get_object()) { + for (auto child2 : child1.value().get_array()) { + for (auto child3 : child2.get_object()) { + for (auto child4 : child3.value().get_array()) { + uint64_t x; + ASSERT_SUCCESS( child4.get(x) ); + ASSERT_EQUAL( x, 9 ); + break; + } + break; + } + break; + } + break; + } + std::cout << " - After breaking out of quadruply-nested arrays and objects ..." << std::endl; + break; + } + + // Test the actual value + case 10: { + ASSERT_EQUAL(key, "actual_value"); + uint64_t actual_value; + ASSERT_SUCCESS( field.value().get(actual_value) ); + ASSERT_EQUAL( actual_value, 10 ); + break; + } + } + + i++; + } + ASSERT_EQUAL( i, 11 ); // Make sure we found all the keys we expected + return true; + })); + return true; + } + + bool iterate_empty_object() { + TEST_START(); + auto json = R"({})"_padded; + + SUBTEST("singlestage::object", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_SUCCESS( doc_result.get(object) ); + for (simdjson_unused auto field : object) { + TEST_FAIL("Unexpected field"); + } + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + simdjson_result object_result = doc_result.get_object(); + for (simdjson_unused auto field : object_result) { + TEST_FAIL("Unexpected field"); + } + return true; + })); + TEST_SUCCEED(); + } + + bool value_search_unescaped_key() { + TEST_START(); + auto json = R"({"k\u0065y": 1})"_padded; + SUBTEST("singlestage::unescapedkey", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + bool got_key = false; + ASSERT_SUCCESS( doc_result.get(object) ); + for (auto field : object) { + std::string_view keyv; + ASSERT_SUCCESS( field.unescaped_key().get(keyv) ); + if(keyv == "key") { + int64_t value; + ASSERT_SUCCESS( field.value().get(value) ); + ASSERT_EQUAL( value, 1); + got_key = true; + } + } + return got_key; + })); + SUBTEST("singlestage::rawkey", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_SUCCESS( doc_result.get(object) ); + bool got_key = false; + for (auto field : object) { + singlestage::raw_json_string keyv; + ASSERT_SUCCESS( field.key().get(keyv) ); + if(keyv == R"(k\u0065y)") { + int64_t value; + ASSERT_SUCCESS( field.value().get(value) ); + ASSERT_EQUAL( value, 1); + got_key = true; + } + } + return got_key; + })); + TEST_SUCCEED(); + } + + bool issue_1480() { + TEST_START(); + auto json = R"({ "name" : "something", "version": "0.13.2", "version_major": 0})"_padded; + + SUBTEST("singlestage::issue_1480::object", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_SUCCESS( doc_result.get(object) ); + ASSERT_EQUAL( object.find_field_unordered("version_major").get_uint64().value_unsafe(), 0 ); + ASSERT_EQUAL( object.find_field_unordered("version").get_string().value_unsafe(), "0.13.2"); + return true; + })); + SUBTEST("singlestage::issue_1480::big-key", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_SUCCESS( doc_result.get(object) ); + std::vector large_buf(512,' '); + std::string_view lb{large_buf.data(), large_buf.size()}; + ASSERT_ERROR( object.find_field(lb), NO_SUCH_FIELD ); + return true; + })); + +#if SIMDJSON_EXCEPTIONS + SUBTEST("singlestage::issue_1480::object-iteration", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_SUCCESS( doc_result.get(object) ); + size_t counter_version{0}; + size_t counter_version_major{0}; + for(auto field : object) { + std::string_view keyv = field.unescaped_key(); + if(keyv == "version") { counter_version++; } + if(keyv == "version_major") { counter_version_major++; } + } + ASSERT_EQUAL( counter_version, 1 ); + ASSERT_EQUAL( counter_version_major, 1 ); + return true; + })); + SUBTEST("singlestage::issue_1480::object-iteration-string-view", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_SUCCESS( doc_result.get(object) ); + size_t counter_version{0}; + size_t counter_version_major{0}; + const char * bufv = "version"; + const char * bufvm = "version_major"; + + std::string_view v{bufv}; + std::string_view vm{bufvm}; + + for(auto field : object) { + std::string_view keyv = field.unescaped_key(); + if(keyv == v) { counter_version++; } + if(keyv == vm) { counter_version_major++; } + } + ASSERT_EQUAL( counter_version, 1 ); + ASSERT_EQUAL( counter_version_major, 1 ); + return true; + })); + SUBTEST("singlestage::issue_1480::original", test_singlestage_doc(json, [&](auto doc_result) { + size_t counter{0}; + for (auto field : doc_result.get_object()) { + auto key = field.key().value(); + if (key == "version") { counter++; } + } + ASSERT_EQUAL( counter, 1 ); + return true; + })); + SUBTEST("singlestage::issue_1480::object-operator", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_SUCCESS( doc_result.get(object) ); + ASSERT_EQUAL( std::string_view(object["version"]), "0.13.2" ); + ASSERT_EQUAL( uint64_t(object["version_major"]), 0 ); + return true; + })); +#endif + TEST_SUCCEED(); + } + +#if SIMDJSON_EXCEPTIONS + + bool iterate_object_exception() { + TEST_START(); + auto json = R"({ "a": 1, "b": 2, "c": 3 })"_padded; + const char* expected_key[] = { "a", "b", "c" }; + const uint64_t expected_value[] = { 1, 2, 3 }; + ASSERT_TRUE(test_singlestage_doc(json, [&](auto doc_result) { + size_t i = 0; + for (singlestage::field field : doc_result.get_object()) { + ASSERT_EQUAL( field.key(), expected_key[i] ); + ASSERT_EQUAL( uint64_t(field.value()), expected_value[i] ); + i++; + } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); + return true; + })); + TEST_SUCCEED(); + } + + bool iterate_empty_object_exception() { + TEST_START(); + auto json = R"({})"_padded; + + ASSERT_TRUE(test_singlestage_doc(json, [&](auto doc_result) { + for (simdjson_unused singlestage::field field : doc_result.get_object()) { + TEST_FAIL("Unexpected field"); + } + return true; + })); + SUBTEST("singlestage::object-document-rewind", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_RESULT( doc_result.type(), json_type::object ); + ASSERT_SUCCESS( doc_result.get(object) ); + size_t i = 0; + for (auto field : object) { + (void)field; + i++; + } + ASSERT_EQUAL( i, 0 ); + doc_result.rewind(); + i = 0; + for (auto field : object) { + (void)field; + i++; + } + ASSERT_EQUAL( i, 0 ); + return true; + })); + SUBTEST("singlestage::object-rewind", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_RESULT( doc_result.type(), json_type::object ); + ASSERT_SUCCESS( doc_result.get(object) ); + size_t i = 0; + for (auto field : object) { + (void)field; + i++; + } + ASSERT_EQUAL( i, 0 ); + object.reset(); + i = 0; + for (auto field : object) { + (void)field; + i++; + } + ASSERT_EQUAL( i, 0 ); + return true; + })); + SUBTEST("singlestage::object-rewind", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::object object; + ASSERT_RESULT( doc_result.type(), json_type::object ); + ASSERT_SUCCESS( doc_result.get(object) ); + size_t i = 0; + for (auto field : object) { + (void)field; + i++; + } + ASSERT_EQUAL( i, 0 ); + object.reset(); + i = 0; + for (auto field : object) { + (void)field; + i++; + } + ASSERT_EQUAL( i, 0 ); + return true; + })); + + TEST_SUCCEED(); + } + +#endif // SIMDJSON_EXCEPTIONS + + bool empty(simdjson::singlestage::object obj) { + bool is_empty; + ASSERT_SUCCESS(obj.is_empty().get(is_empty)); + ASSERT_TRUE(is_empty); + return true; + } + bool value_to_object(simdjson::singlestage::value val) { + singlestage::json_type t; + ASSERT_SUCCESS(val.type().get(t)); + ASSERT_EQUAL(t, singlestage::json_type::object); + simdjson::singlestage::object obj; + ASSERT_SUCCESS(val.get_object().get(obj)); + if(empty(obj) != true) { return false; } + return true; + } + bool empty_rewind_convoluted() { + TEST_START(); + const auto json = R"( {} )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::value val; + ASSERT_SUCCESS(doc.get_value().get(val)); + if(!value_to_object(val)) { return false; } + TEST_SUCCEED(); + } +#if SIMDJSON_EXCEPTIONS + bool value_to_object_except(simdjson::singlestage::value val) { + singlestage::json_type t = val.type(); + ASSERT_EQUAL(t, singlestage::json_type::object); + if(empty(simdjson::singlestage::object(val)) != true) { return false; } + return true; + } + bool empty_rewind_convoluted_with_exceptions() { + TEST_START(); + const auto json = R"( {} )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::value val; + ASSERT_SUCCESS(doc.get_value().get(val)); + if(value_to_object_except(val) != true) { return false; } + TEST_SUCCEED(); + } +#endif + + bool iterate_empty_object_count() { + TEST_START(); + auto json = R"( {} )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::object obj; + size_t count; + ASSERT_SUCCESS(doc.get_object().get(obj)); + ASSERT_SUCCESS(obj.count_fields().get(count)); + ASSERT_EQUAL(count, 0); + TEST_SUCCEED(); + } + + bool issue1876a() { + TEST_START(); + auto json = R"( {} )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::object obj; + ASSERT_SUCCESS(doc.get_object().get(obj)); + size_t count; + ASSERT_SUCCESS(obj.count_fields().get(count)); + ASSERT_EQUAL(count, 0); + for (auto field : obj) { + std::string_view key; + ASSERT_SUCCESS(field.unescaped_key().get(key)); + double value; + ASSERT_SUCCESS(field.value().get_double().get(value)); + std::cout << key << " " << value << std::endl; + } + TEST_SUCCEED(); + } + + bool issue1876() { + TEST_START(); + auto json = R"( {} )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + size_t count; + ASSERT_SUCCESS(doc.count_fields().get(count)); + singlestage::object obj; + ASSERT_SUCCESS(doc.get_object().get(obj)); + + ASSERT_EQUAL(count, 0); + for (auto field : obj) { + std::string_view key; + ASSERT_SUCCESS(field.unescaped_key().get(key)); + double value; + ASSERT_SUCCESS(field.value().get_double().get(value)); + std::cout << key << " " << value << std::endl; + } + TEST_SUCCEED(); + } + + bool issue1742() { + TEST_START(); + auto json = R"( { + "code": 0, + "method": "subscribe", + "result": { + "instrument_name": "DAI_USDC", + "subscription": "trade.DAI_USDC", + "channel": "trade", + "data": [ + { + "dataTime": 1635315736764, + "d": 1928673075051012900, + "s": "SELL", + "p": 1.0004, + "q": 0.01, + "t": 1635315736763, + "i": "DAI_USDC" + } + ] + } +} )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + simdjson::singlestage::array data; + ASSERT_SUCCESS(doc["result"]["data"].get_array().get(data)); + for (auto d : data) { + simdjson::singlestage::object obj; + ASSERT_SUCCESS(d.get_object().get(obj)); + size_t count; + ASSERT_SUCCESS(obj.count_fields().get(count)); + ASSERT_EQUAL(count, 7); + } + TEST_SUCCEED(); + } + + + bool issue1742_value() { + TEST_START(); + auto json = R"( { + "code": 0, + "method": "subscribe", + "result": { + "instrument_name": "DAI_USDC", + "subscription": "trade.DAI_USDC", + "channel": "trade", + "data": [ + { + "dataTime": 1635315736764, + "d": 1928673075051012900, + "s": "SELL", + "p": 1.0004, + "q": 0.01, + "t": 1635315736763, + "i": "DAI_USDC" + } + ] + } +} )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + simdjson::singlestage::array data; + ASSERT_SUCCESS(doc["result"]["data"].get_array().get(data)); + for (auto d : data) { + size_t count; + ASSERT_SUCCESS(d.count_fields().get(count)); + ASSERT_EQUAL(count, 7); + } + TEST_SUCCEED(); + } + + bool iterate_basic_object_count() { + TEST_START(); + auto json = R"( {"a":-55, "b":3.23, "c":100000000000000000000, "d":true, "e":null} )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::object obj; + size_t count; + ASSERT_SUCCESS(doc.get_object().get(obj)); + ASSERT_SUCCESS(obj.count_fields().get(count)); + ASSERT_EQUAL(count, 5); + TEST_SUCCEED(); + } + + bool iterate_complex_object_count() { + TEST_START(); + auto json = R"( { + "first": {"a":1, "b":[1,2,3], "c":{}}, + "second":[true, false, null, {"1":[4,5,6], "2":3.14}] + } )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::object obj1; + size_t count{0}; + ASSERT_SUCCESS(doc.get_object().get(obj1)); + ASSERT_SUCCESS(obj1.count_fields().get(count)); + ASSERT_EQUAL(count, 2); + count = 0; + singlestage::object obj2; + ASSERT_SUCCESS(doc.find_field("first").get_object().get(obj2)); + ASSERT_SUCCESS(obj2.count_fields().get(count)); + ASSERT_EQUAL(count, 3); + count = 0; + singlestage::object obj3; + ASSERT_SUCCESS(obj2.find_field("c").get_object().get(obj3)); + ASSERT_SUCCESS(obj3.count_fields().get(count)); + ASSERT_EQUAL(count, 0); + count = 0; + singlestage::object obj4; + ASSERT_SUCCESS(doc.at_pointer("/second/3").get_object().get(obj4)); + ASSERT_SUCCESS(obj4.count_fields().get(count)); + ASSERT_EQUAL(count, 2); + TEST_SUCCEED(); + } + + bool iterate_bad_object_count() { + TEST_START(); + padded_string bad_jsons[3] = {R"( {"a":5 "b":3} )"_padded, R"( {"a":5, 3} )"_padded, R"( {"a":5, "b": } )"_padded}; + std::string names[3] = {"missing_comma", "missing_key", "missing_value"}; + size_t count{0}; + + for (auto name : names) { + SUBTEST("singlestage::" + name, test_singlestage_doc(bad_jsons[count++], [&](auto doc_result) { + singlestage::object object; + ASSERT_RESULT( doc_result.type(), json_type::object ); + ASSERT_SUCCESS( doc_result.get(object) ); + ASSERT_ERROR(object.count_fields(), TAPE_ERROR); + return true; + })); + } + ASSERT_EQUAL(count, 3); + TEST_SUCCEED(); + } + + bool iterate_doc_object_count() { + TEST_START(); + auto empty = R"( {} )"_padded; + SUBTEST("singlestage::empty_doc_object", test_singlestage_doc(empty, [&](auto doc_result) { + size_t count; + ASSERT_RESULT( doc_result.type(), json_type::object ); + ASSERT_SUCCESS( doc_result.count_fields().get(count) ); + ASSERT_EQUAL( count, 0 ); + return true; + })); + auto basic = R"( {"a":-1.234, "b":false, "c":null, "d":[1000.1,-2000.2,3000.3], "e":{"a":true, "b":false}} )"_padded; + SUBTEST("singlestage::basic_doc_object", test_singlestage_doc(basic, [&](auto doc_result) { + size_t count; + ASSERT_RESULT( doc_result.type(), json_type::object ); + ASSERT_SUCCESS( doc_result.count_fields().get(count) ); + ASSERT_EQUAL( count, 5 ); + return true; + })); + TEST_SUCCEED(); + } + + + bool issue1974a() { + TEST_START(); + padded_string bad_json = R"({"key":111)"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(bad_json).get(doc)); + singlestage::object object; + ASSERT_ERROR(doc.get_object().get(object), INCOMPLETE_ARRAY_OR_OBJECT); + TEST_SUCCEED(); + } + + bool issue1974b() { + TEST_START(); + padded_string bad_json = R"({"key":111)"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(bad_json).get(doc)); + singlestage::value val; + ASSERT_ERROR(doc.get_value().get(val), INCOMPLETE_ARRAY_OR_OBJECT); + TEST_SUCCEED(); + } + + bool iterate_bad_doc_object_count() { + TEST_START(); + padded_string bad_jsons[4] = {R"( {"a":5 "b":3} )"_padded, R"( {"a":5, 3} )"_padded, R"( {"a":5, "b": } )"_padded, R"( {"a":5, "b":3 )"_padded}; + std::string names[4] = {"missing_comma", "missing_key", "missing_value", "missing_bracket"}; + simdjson::error_code errors[4] = {TAPE_ERROR, TAPE_ERROR, TAPE_ERROR, INCOMPLETE_ARRAY_OR_OBJECT}; + size_t count{0}; + + for (auto name : names) { + SUBTEST("singlestage::" + name, test_singlestage_doc(bad_jsons[count], [&](auto doc_result) { + ASSERT_RESULT(doc_result.type(), json_type::object ); + ASSERT_ERROR(doc_result.count_fields(), errors[count]); + return true; + })); + count++; + } + ASSERT_EQUAL(count, 4); + TEST_SUCCEED(); + } + + + bool test_strager() { + // Find exactly one of a/myint or b/myint, if both or none are present, + // return false. + auto process = [](singlestage::object root) -> std::pair { + /////////////////////////////////////////////////////////// + // The core idea with On-Demand is that you should visit everything only + // once. + // + // Thus, with On-Demand, you must consume a field once you have accessed it. + // Thus if you do root["a"], you must full consume the corresponding + // value (if present) before you move to another field (e.g., root["a"]). + // + // If, somehow, you only want the value corresponding to "a" if some + // other key is not present, then you must still parse/consume the value + // corresponding to "a", and later throw the result away. + /////////////////////////////////////////////////////////// + uint64_t myint; + // Notice in the line below that I use chaining... I do not + // first convert root["a"] to an object and then seek + // "myint" into. Chaining works in this instance because + // I only need to access the object 'root["a"]' for a single + // key, if I needed to do more work on root["a"], then I should + // create an object. + if(root["a"]["myint"].get(myint) == SUCCESS) { + // We want myint to be discarded if "b" is also present. + if(! root["b"].error() ) { + // Oops, the other key is also present!!! We must throw + // away 'myint' and return false. + // Notice how I do not parse the value corresponding to + // '"b"', as it is unneeded. + return std::make_pair(false, 0); + } + // All is good, + return std::make_pair(true,myint); + } + // Now we know that "a" is not present, so seek "b". + if(root["b"]["myint"].get(myint) == SUCCESS) { + return std::make_pair(true, myint); + } + // None are present. + return std::make_pair(false, 0); + }; + + TEST_START(); + + { + singlestage::parser parser; + padded_string json = R"({ "a": { "myint": 42 } })"_padded; + std::cout << " - Subtest 'a' - JSON: " << (json) << " ..." << std::endl; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::object obj; + ASSERT_SUCCESS(doc.get_object().get(obj)); + std::pair ret = process(obj); + ASSERT_TRUE(ret.first); + ASSERT_EQUAL(ret.second, 42); + } + + { + singlestage::parser parser; + padded_string json = R"({ "b": { "myint": 42 } })"_padded; + std::cout << " - Subtest 'b' - JSON: " << (json) << " ..." << std::endl; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::object obj; + ASSERT_SUCCESS(doc.get_object().get(obj)); + std::pair ret = process(obj); + ASSERT_TRUE(ret.first); + ASSERT_EQUAL(ret.second, 42); + } + + + { + singlestage::parser parser; + padded_string json = R"({ "b": { "myint": 42 }, "a": { "myint": 42 } })"_padded; + std::cout << " - Subtest 'b&a' - JSON: " << (json) << " ..." << std::endl; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::object obj; + ASSERT_SUCCESS(doc.get_object().get(obj)); + std::pair ret = process(obj); + ASSERT_FALSE(ret.first); + ASSERT_EQUAL(ret.second, 0); + } + + { + singlestage::parser parser; + padded_string json = R"({ "z": { "myint": 42 }, "y": { "myint": 42 } })"_padded; + std::cout << " - Subtest 'none' - JSON: " << (json) << " ..." << std::endl; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + singlestage::object obj; + ASSERT_SUCCESS(doc.get_object().get(obj)); + std::pair ret = process(obj); + ASSERT_FALSE(ret.first); + ASSERT_EQUAL(ret.second, 0); + } + + TEST_SUCCEED(); + } + + bool run() { + return issue1979() && + issue1977() && +#if SIMDJSON_EXCEPTIONS + issue1965() && +#endif + issue1974a() && + issue1974b() && + issue1876a() && + issue1876() && + test_strager() && + issue1745() && + issue1742() && + issue1742_value() && + issue1723() && + value_search_unescaped_key() && + missing_key_continue() && + no_missing_keys() && + missing_keys() && + missing_keys_for_empty_top_level_object() && +#if SIMDJSON_EXCEPTIONS + issue1745_with_exceptions() && + issue1723_except() && + fixed_broken_issue_1521() && + issue_1521() && + broken_issue_1521() && +#endif + iterate_object() && + iterate_empty_object() && + iterate_object_partial_children() && + issue_1480() && + empty_rewind_convoluted() && +#if SIMDJSON_EXCEPTIONS + iterate_object_exception() && + empty_rewind_convoluted_with_exceptions() && +#endif // SIMDJSON_EXCEPTIONS + iterate_empty_object_count() && + iterate_basic_object_count() && + iterate_complex_object_count() && + iterate_bad_object_count() && + iterate_bad_doc_object_count() && + true; + } + +} // namespace object_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, object_tests::run); +} diff --git a/tests/singlestage/singlestage_ordering_tests.cpp b/tests/singlestage/singlestage_ordering_tests.cpp new file mode 100644 index 000000000..23cae5cb7 --- /dev/null +++ b/tests/singlestage/singlestage_ordering_tests.cpp @@ -0,0 +1,167 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace ordering_tests { + using namespace std; + +#if SIMDJSON_EXCEPTIONS + + auto json = "{\"coordinates\":[{\"x\":1.1,\"y\":2.2,\"z\":3.3}]}"_padded; + + bool in_order_object_index() { + TEST_START(); + singlestage::parser parser{}; + auto doc = parser.iterate(json); + double x{0}; + double y{0}; + double z{0}; + for (singlestage::object point_object : doc["coordinates"]) { + x += double(point_object["x"]); + y += double(point_object["y"]); + z += double(point_object["z"]); + } + return (x == 1.1) && (y == 2.2) && (z == 3.3); + } + + bool in_order_object_find_field_unordered() { + TEST_START(); + singlestage::parser parser{}; + auto doc = parser.iterate(json); + double x{0}; + double y{0}; + double z{0}; + for (singlestage::object point_object : doc["coordinates"]) { + x += double(point_object.find_field_unordered("x")); + y += double(point_object.find_field_unordered("y")); + z += double(point_object.find_field_unordered("z")); + } + return (x == 1.1) && (y == 2.2) && (z == 3.3); + } + + bool in_order_object_find_field() { + TEST_START(); + singlestage::parser parser{}; + auto doc = parser.iterate(json); + double x{0}; + double y{0}; + double z{0}; + for (singlestage::object point_object : doc["coordinates"]) { + x += double(point_object.find_field("x")); + y += double(point_object.find_field("y")); + z += double(point_object.find_field("z")); + } + return (x == 1.1) && (y == 2.2) && (z == 3.3); + } + + bool out_of_order_object_index() { + TEST_START(); + singlestage::parser parser{}; + auto doc = parser.iterate(json); + double x{0}; + double y{0}; + double z{0}; + for (singlestage::object point_object : doc["coordinates"]) { + z += double(point_object["z"]); + x += double(point_object["x"]); + y += double(point_object["y"]); + } + return (x == 1.1) && (y == 2.2) && (z == 3.3); + } + + bool out_of_order_object_find_field_unordered() { + TEST_START(); + singlestage::parser parser{}; + auto doc = parser.iterate(json); + double x{0}; + double y{0}; + double z{0}; + for (singlestage::object point_object : doc["coordinates"]) { + z += double(point_object.find_field_unordered("z")); + x += double(point_object.find_field_unordered("x")); + y += double(point_object.find_field_unordered("y")); + } + return (x == 1.1) && (y == 2.2) && (z == 3.3); + } + + bool out_of_order_object_find_field() { + TEST_START(); + singlestage::parser parser{}; + auto doc = parser.iterate(json); + double x{0}; + double y{0}; + double z{0}; + for (singlestage::object point_object : doc["coordinates"]) { + z += double(point_object.find_field("z")); + ASSERT_ERROR( point_object.find_field("x"), NO_SUCH_FIELD ); + ASSERT_ERROR( point_object.find_field("y"), NO_SUCH_FIELD ); + } + return (x == 0) && (y == 0) && (z == 3.3); + } + + bool foreach_object_field_lookup() { + TEST_START(); + singlestage::parser parser{}; + auto doc = parser.iterate(json); + double x{0}; + double y{0}; + double z{0}; + for (auto point_object : doc["coordinates"]) { + for (auto field : point_object.get_object()) { + if (field.key() == "z") { z += double(field.value()); } + else if (field.key() == "x") { x += double(field.value()); } + else if (field.key() == "y") { y += double(field.value()); } + } + } + return (x == 1.1) && (y == 2.2) && (z == 3.3); + } + + bool use_values_out_of_order_after_array() { + TEST_START(); + singlestage::parser parser{}; + auto doc = parser.iterate(json); + simdjson_result x{}, y{}, z{}; + for (auto point_object : doc["coordinates"]) { + x = point_object["x"]; + y = point_object["y"]; + z = point_object["z"]; + } + return (double(x) == 1.1) && (double(z) == 3.3) && (double(y) == 2.2); + } + + bool use_object_multiple_times_out_of_order() { + TEST_START(); + singlestage::parser parser{}; + auto json2 = "{\"coordinates\":{\"x\":1.1,\"y\":2.2,\"z\":3.3}}"_padded; + auto doc = parser.iterate(json2); + auto x = doc["coordinates"]["x"]; + auto y = doc["coordinates"]["y"]; + auto z = doc["coordinates"]["z"]; + return (double(x) == 1.1) && (double(z) == 3.3) && (double(y) == 2.2); + } + +#endif // SIMDJSON_EXCEPTIONS + + bool run() { + return +#if SIMDJSON_EXCEPTIONS + in_order_object_index() && + in_order_object_find_field_unordered() && + in_order_object_find_field() && + out_of_order_object_index() && + out_of_order_object_find_field_unordered() && + out_of_order_object_find_field() && + foreach_object_field_lookup() && + use_values_out_of_order_after_array() && + use_object_multiple_times_out_of_order() && +#endif // SIMDJSON_EXCEPTIONS + true; + } + +} // namespace ordering_tests + + +int main(int argc, char *argv[]) { + return test_main(argc, argv, ordering_tests::run); +} diff --git a/tests/singlestage/singlestage_parse_api_tests.cpp b/tests/singlestage/singlestage_parse_api_tests.cpp new file mode 100644 index 000000000..c9e946fbc --- /dev/null +++ b/tests/singlestage/singlestage_parse_api_tests.cpp @@ -0,0 +1,278 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace parse_api_tests { + using namespace std; + + const padded_string BASIC_JSON = "[1,2,3]"_padded; + const padded_string BASIC_NDJSON = "[1,2,3]\n[4,5,6]"_padded; + const padded_string EMPTY_NDJSON = ""_padded; + + + bool parser_iterate_empty() { + TEST_START(); + FILE *p; + // Of course, we could just call iterate on the empty string, but + // we want to test the whole process. + const char *const tmpfilename = "emptysinglestage.txt"; + if((p = fopen(tmpfilename, "w")) != nullptr) { + fclose(p); + auto json = padded_string::load(tmpfilename); + singlestage::document doc; + singlestage::parser parser; + auto error = parser.iterate(json).get(doc); + remove(tmpfilename); + if(error != simdjson::EMPTY) { + std::cerr << "Was expecting empty but got " << error << std::endl; + return false; + } + } else { + std::cout << "Warning: I could not create temporary file " << tmpfilename << std::endl; + std::cout << "We omit testing the empty file case." << std::endl; + } + TEST_SUCCEED(); + } + + bool parser_iterate() { + TEST_START(); + singlestage::parser parser; + auto doc = parser.iterate(BASIC_JSON); + ASSERT_SUCCESS( doc.get_array() ); + return true; + } + + bool parser_iterate_padded() { + TEST_START(); + singlestage::parser parser; + const char json_str[] = "12\0 ";// 64 bytes of padding + ASSERT_EQUAL(sizeof(json_str), 66); + ASSERT_EQUAL(strlen(json_str), 2); + + { + cout << "- char*" << endl; + auto doc = parser.iterate(json_str, strlen(json_str), sizeof(json_str)); + ASSERT_SUCCESS( doc.get_double() ); + } + + { + cout << "- uint8_t*" << endl; + const uint8_t* json = reinterpret_cast(json_str); + auto doc = parser.iterate(json, strlen(json_str), sizeof(json_str)); + ASSERT_SUCCESS( doc.get_double() ); + } + + { + cout << "- string_view" << endl; + std::string_view json(json_str); + auto doc = parser.iterate(json, sizeof(json_str)); + ASSERT_SUCCESS( doc.get_double() ); + } + + { + cout << "- string" << endl; + std::string json = "12"; + json.reserve(json.length() + SIMDJSON_PADDING); + auto doc = parser.iterate(json); + ASSERT_SUCCESS( doc.get_double() ); + } + + TEST_SUCCEED(); + } + + bool parser_iterate_padded_string_view() { + TEST_START(); + singlestage::parser parser; + const char json_str[] = "12\0 "; // 64 bytes of padding + ASSERT_EQUAL(sizeof(json_str), 66); + ASSERT_EQUAL(strlen(json_str), 2); + + { + cout << "- padded_string_view(string_view)" << endl; + padded_string_view json(std::string_view(json_str), sizeof(json_str)); + auto doc = parser.iterate(json); + ASSERT_SUCCESS( doc.get_double() ); + } + + { + cout << "- padded_string_view(char*)" << endl; + auto doc = parser.iterate(padded_string_view(json_str, strlen(json_str), sizeof(json_str))); + ASSERT_SUCCESS( doc.get_double() ); + } + + { + cout << "- padded_string_view(string)" << endl; + std::string json = "12"; + json.reserve(json.length() + SIMDJSON_PADDING); + auto doc = parser.iterate(padded_string_view(json)); + ASSERT_SUCCESS( doc.get_double() ); + } + + { + cout << "- padded_string_view(string_view(char*))" << endl; + padded_string_view json(json_str, sizeof(json_str)); + auto doc = parser.iterate(json); + ASSERT_SUCCESS( doc.get_double() ); + } + + TEST_SUCCEED(); + } + + bool parser_iterate_insufficient_padding() { + TEST_START(); + singlestage::parser parser; + constexpr char json_str[] = "12\0 "; // 63 bytes of padding + ASSERT_EQUAL(sizeof(json_str), 65); + ASSERT_EQUAL(strlen(json_str), 2); + ASSERT_EQUAL(padded_string_view(json_str, strlen(json_str), sizeof(json_str)).padding(), 63); + ASSERT_EQUAL(SIMDJSON_PADDING, 64); + + { + cout << "- char*, 63 padding" << endl; + ASSERT_ERROR( parser.iterate(json_str, strlen(json_str), sizeof(json_str)), INSUFFICIENT_PADDING ); + cout << "- char*, 0 padding" << endl; + ASSERT_ERROR( parser.iterate(json_str, strlen(json_str), strlen(json_str)), INSUFFICIENT_PADDING ); + } + + { + std::string_view json(json_str); + cout << "- string_view, 63 padding" << endl; + ASSERT_ERROR( parser.iterate(json, sizeof(json_str)), INSUFFICIENT_PADDING ); + cout << "- string_view, 0 padding" << endl; + ASSERT_ERROR( parser.iterate(json, strlen(json_str)), INSUFFICIENT_PADDING ); + } + + { + std::string json = "12"; + json.shrink_to_fit(); + cout << "- string, 0 padding" << endl; + ASSERT_ERROR( parser.iterate(json), INSUFFICIENT_PADDING ); + // It's actually kind of hard to allocate "just enough" capacity, since the string tends + // to grow more than you tell it to. + } + + TEST_SUCCEED(); + } + +#if SIMDJSON_EXCEPTIONS + bool parser_iterate_exception() { + TEST_START(); + singlestage::parser parser; + auto doc = parser.iterate(BASIC_JSON); + simdjson_unused singlestage::array array = doc; + TEST_SUCCEED(); + } + + bool parser_document_reuse() { + TEST_START(); + singlestage::document doc; + // A document spans about 40 bytes. Nevertheless, some users + // would rather reuse them. + std::cout << sizeof(doc) << std::endl; + auto json = R"({"key": "value"})"_padded; + auto jsonbad = R"({"key": "value")"_padded; // deliberaty broken + auto jsonunclosedstring = "{\"coordinates:[{\"x\":1.1,\"y\":2.2,\"z\":3.3}]}"_padded; + std::string_view output; + + singlestage::parser parser; + std::cout << "correct document (1)" << std::endl; + + ASSERT_SUCCESS( parser.iterate(json).get(doc) ); + + ASSERT_SUCCESS(simdjson::to_json_string(doc).get(output)); + std::cout << output << std::endl; + + std::cout << "correct document (2)" << std::endl; + + ASSERT_SUCCESS( parser.iterate(json).get(doc) ); + for(singlestage::field field : doc.get_object() ) { + std::cout << "field: " << field.key() << std::endl; + } + std::cout << "unclosed string document " << std::endl; + simdjson::error_code error; + if((error = parser.iterate(jsonunclosedstring).get(doc)) == SUCCESS) { + // fallback kernel: + ASSERT_EQUAL( doc.get_object().find_field("coordinates").error(), TAPE_ERROR ); + } else { + // regular kernels: + ASSERT_EQUAL( error, UNCLOSED_STRING ); + } + + std::cout << "truncated document " << std::endl; + ASSERT_SUCCESS( parser.iterate(jsonbad).get(doc) ); + ASSERT_EQUAL( simdjson::to_json_string(doc).get(output), TAPE_ERROR ); + + std::cout << "correct document with new doc" << std::endl; + singlestage::document doc2; + ASSERT_SUCCESS( parser.iterate(json).get(doc2) ); + for(singlestage::field field : doc2.get_object() ) { + std::cout << "field: " << field.key() << std::endl; + } + std::cout << "correct document (3): " << doc.to_debug_string() << std::endl; + + std::cout << "correct document (3)" << std::endl; + ASSERT_SUCCESS( parser.iterate(json).get(doc) ); + std::cout << doc.to_debug_string() << std::endl; + for(singlestage::field field : doc.get_object() ) { + std::cout << "field: " << field.key() << std::endl; + } + + std::cout << "unclosed string document " << std::endl; + ASSERT_SUCCESS( parser.iterate(jsonbad).get(doc) ); + ASSERT_EQUAL( simdjson::to_json_string(doc).get(output), TAPE_ERROR ); + + // next two lines are terrible code. + doc.~document(); + doc = singlestage::document(); + // + + std::cout << "correct document (4)" << std::endl; + + ASSERT_SUCCESS( parser.iterate(json).get(doc) ); + ASSERT_SUCCESS( simdjson::to_json_string(doc).get(output) ); + std::cout << output << std::endl; + + std::cout << "unclosed string document " << std::endl; + + if((error = parser.iterate(jsonunclosedstring).get(doc)) == SUCCESS) { + // fallback kernel: + ASSERT_EQUAL( doc.get_object().find_field("coordinates").error(), TAPE_ERROR ); + } else { + // regular kernels: + ASSERT_EQUAL( error, UNCLOSED_STRING ); + } + + + // next two lines are terrible code. + doc.~document(); + doc = singlestage::document(); + // + std::cout << "correct document (5)" << std::endl; + + ASSERT_SUCCESS( parser.iterate(json).get(doc) ); + ASSERT_SUCCESS( simdjson::to_json_string(doc).get(output) ); + std::cout << output << std::endl; + + TEST_SUCCEED(); + } +#endif // SIMDJSON_EXCEPTIONS + + bool run() { + return parser_iterate_empty() && + parser_iterate() && + parser_iterate_padded() && + parser_iterate_padded_string_view() && + parser_iterate_insufficient_padding() && +#if SIMDJSON_EXCEPTIONS + parser_document_reuse() && + parser_iterate_exception() && +#endif // SIMDJSON_EXCEPTIONS + true; + } +} + + +int main(int argc, char *argv[]) { + return test_main(argc, argv, parse_api_tests::run); +} diff --git a/tests/singlestage/singlestage_readme_examples.cpp b/tests/singlestage/singlestage_readme_examples.cpp new file mode 100644 index 000000000..48cbbde65 --- /dev/null +++ b/tests/singlestage/singlestage_readme_examples.cpp @@ -0,0 +1,1365 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace std; +using namespace simdjson; +using error_code=simdjson::error_code; + +bool string1() { + const char * data = "my data"; // 7 bytes + simdjson::padded_string my_padded_data(data, 7); // copies to a padded buffer + std::cout << my_padded_data << std::endl; + return true; +} + +bool string2() { + std::string data = "my data"; + simdjson::padded_string my_padded_data(data); // copies to a padded buffer + std::cout << my_padded_data << std::endl; + return true; +} + + +#if SIMDJSON_EXCEPTIONS +bool gen_raw1() { + TEST_START(); + simdjson::singlestage::parser parser; + simdjson::padded_string docdata = R"({"value":123})"_padded; + simdjson::singlestage::document doc = parser.iterate(docdata); + simdjson::singlestage::object obj = doc.get_object(); + string_view token = obj.raw_json(); // gives you `{"value":123}` + ASSERT_EQUAL(token, R"({"value":123})"); + TEST_SUCCEED(); +} + +bool gen_raw2() { + TEST_START(); + simdjson::singlestage::parser parser; + simdjson::padded_string docdata = R"([1,2,3])"_padded; + simdjson::singlestage::document doc = parser.iterate(docdata); + simdjson::singlestage::array arr = doc.get_array(); + string_view token = arr.raw_json(); // gives you `[1,2,3]` + ASSERT_EQUAL(token, R"([1,2,3])"); + TEST_SUCCEED(); +} + +bool gen_raw3() { + TEST_START(); + simdjson::singlestage::parser parser; + simdjson::padded_string docdata = R"({"value":123})"_padded; + simdjson::singlestage::document doc = parser.iterate(docdata); + simdjson::singlestage::object obj = doc.get_object(); + string_view token = obj.raw_json(); // gives you `{"value":123}` + ASSERT_EQUAL(token, R"({"value":123})"); + obj.reset(); // revise the object + uint64_t x = obj["value"]; // gives me 123 + ASSERT_EQUAL(x, 123); + + TEST_SUCCEED(); +} + +bool at_end() { + TEST_START(); + auto json = R"([1, 2] foo ])"_padded; + singlestage::parser parser; + singlestage::document doc = parser.iterate(json); + singlestage::array array = doc.get_array(); + for (uint64_t values : array) { + std::cout << values << std::endl; + } + if(!doc.at_end()) { + std::cerr << "trailing content at byte index " << doc.current_location() - json.data() << std::endl; + } + TEST_SUCCEED(); +} + +bool number_tests() { + TEST_START(); + singlestage::parser parser; + padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999])"_padded; + singlestage::document doc = parser.iterate(docdata); + singlestage::array arr = doc.get_array(); + for(singlestage::value val : arr) { + std::cout << val << " "; + std::cout << "negative: " << val.is_negative() << " "; + std::cout << "is_integer: " << val.is_integer() << " "; + singlestage::number num = val.get_number(); + singlestage::number_type t = num.get_number_type(); + // direct computation without materializing the number: + singlestage::number_type dt = val.get_number_type(); + if(t != dt) { throw std::runtime_error("bug"); } + switch(t) { + case singlestage::number_type::signed_integer: + std::cout << "integer: " << int64_t(num) << " "; + std::cout << "integer: " << num.get_int64() << std::endl; + break; + case singlestage::number_type::unsigned_integer: + std::cout << "large 64-bit integer: " << uint64_t(num) << " "; + std::cout << "large 64-bit integer: " << num.get_uint64() << std::endl; + break; + case singlestage::number_type::floating_point_number: + std::cout << "float: " << double(num) << " "; + std::cout << "float: " << num.get_double() << std::endl; + break; + } + } + TEST_SUCCEED(); +} + +bool recursive_print_json(singlestage::value element) { + TEST_START(); + bool add_comma; + switch (element.type()) { + case singlestage::json_type::array: + cout << "["; + add_comma = false; + for (auto child : element.get_array()) { + if (add_comma) { + cout << ","; + } + // We need the call to value() to get + // an singlestage::value type. + recursive_print_json(child.value()); + add_comma = true; + } + cout << "]"; + break; + case singlestage::json_type::object: + cout << "{"; + add_comma = false; + for (auto field : element.get_object()) { + if (add_comma) { + cout << ","; + } + // key() returns the key as it appears in the raw + // JSON document, if we want the unescaped key, + // we should do field.unescaped_key(). + cout << "\"" << field.key() << "\": "; + recursive_print_json(field.value()); + add_comma = true; + } + cout << "}\n"; + break; + case singlestage::json_type::number: + // assume it fits in a double + cout << element.get_double(); + break; + case singlestage::json_type::string: + // get_string() would return escaped string, but + // we are happy with unescaped string. + cout << "\"" << element.get_raw_json_string() << "\""; + break; + case singlestage::json_type::boolean: + cout << element.get_bool(); + break; + case singlestage::json_type::null: + // we check that the value is indeed null + // otherwise: an error is thrown. + if(element.is_null()) { + cout << "null"; + } + break; + } + TEST_SUCCEED(); +} + +bool basics_treewalk() { + TEST_START(); + padded_string json[3] = {R"( [ + { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, + { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, + { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] } + ] )"_padded, R"( {"key":"value"} )"_padded, "[12,3]"_padded}; + singlestage::parser parser; + for(size_t i = 0 ; i < 3; i++) { + singlestage::document doc = parser.iterate(json[i]); + singlestage::value val = doc; + recursive_print_json(val); + std::cout << std::endl; + } + TEST_SUCCEED(); +} + +bool print_depth_space(singlestage::value element) { + TEST_START(); + for(auto i = 0; i < element.current_depth(); i++) { + cout << " "; + } + TEST_SUCCEED(); +} + +bool recursive_print_json_breakline(singlestage::value element) { + TEST_START(); + bool add_comma; + switch (element.type()) { + case singlestage::json_type::array: + cout << endl; + print_depth_space(element); + cout << "["; + add_comma = false; + for (auto child : element.get_array()) { + if (add_comma) { + print_depth_space(element); + cout << ","; + } + // We need the call to value() to get + // an singlestage::value type. + recursive_print_json_breakline(child.value()); + add_comma = true; + } + cout << "]"; + break; + case singlestage::json_type::object: + cout << endl; + print_depth_space(element); + cout << "{"; + add_comma = false; + for (auto field : element.get_object()) { + if (add_comma) { + print_depth_space(element); + cout << ","; + } + // key() returns the key as it appears in the raw + // JSON document, if we want the unescaped key, + // we should do field.unescaped_key(). + cout << "\"" << field.key() << "\": "; + recursive_print_json_breakline(field.value()); + add_comma = true; + } + cout << "}\n"; + break; + case singlestage::json_type::number: + // assume it fits in a double + cout << element.get_double(); + break; + case singlestage::json_type::string: + // get_string() would return escaped string, but + // we are happy with unescaped string. + cout << "\"" << element.get_raw_json_string() << "\""; + break; + case singlestage::json_type::boolean: + cout << element.get_bool(); + break; + case singlestage::json_type::null: + // We check that the value is indeed null + // otherwise: an error is thrown. + if(element.is_null()) { + cout << "null"; + } + break; + } + TEST_SUCCEED(); +} + +bool basics_treewalk_breakline() { + TEST_START(); + padded_string json[3] = {R"( [ + { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, + { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, + { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] } + ] )"_padded, R"( {"key":"value"} )"_padded, "[12,3]"_padded}; + singlestage::parser parser; + for(size_t i = 0 ; i < 3; i++) { + singlestage::document doc = parser.iterate(json[i]); + singlestage::value val = doc; + recursive_print_json_breakline(val); + std::cout << std::endl; + } + TEST_SUCCEED(); +} + +bool basics_1() { + TEST_START(); + + singlestage::parser parser; + auto json = padded_string::load("twitter.json"); + singlestage::document doc = parser.iterate(json); // load and parse a file + + simdjson_unused auto unused_doc = doc.get_object(); + + TEST_SUCCEED(); +} + +bool basics_2() { + TEST_START(); + + singlestage::parser parser; + auto json = "[1,2,3]"_padded; // The _padded suffix creates a simdjson::padded_string instance + singlestage::document doc = parser.iterate(json); // parse a string + + simdjson_unused auto unused_doc = doc.get_array(); + + TEST_SUCCEED(); +} + +bool basics_3() { + TEST_START(); + + singlestage::parser parser; + char json[3+SIMDJSON_PADDING]; + strcpy(json, "[1]"); + singlestage::document doc = parser.iterate(json, strlen(json), sizeof(json)); + + simdjson_unused auto unused_doc = doc.get_array(); + + TEST_SUCCEED(); +} + + +bool json_array_with_array_count() { + TEST_START(); + singlestage::parser parser; + auto cars_json = R"( [ 40.1, 39.9, 37.7, 40.4 ] )"_padded; + auto doc = parser.iterate(cars_json); + auto arr = doc.get_array(); + size_t count = arr.count_elements(); + ASSERT_EQUAL(4, count); + std::cout << count << std::endl; + // We deliberately do it twice: + count = arr.count_elements(); + ASSERT_EQUAL(4, count); + std::cout << count << std::endl; + // Next, we check whether we can iterate normally: + std::vector values(count); + size_t index = 0; + for(double x : arr) { values[index++] = x; } + ASSERT_EQUAL(index, count); + TEST_SUCCEED(); +} + +bool json_value_with_array_count() { + TEST_START(); + singlestage::parser parser; + auto cars_json = R"( {"array":[ 40.1, 39.9, 37.7, 40.4 ]} )"_padded; + auto doc = parser.iterate(cars_json); + auto val = doc["array"]; + size_t count = val.count_elements(); + ASSERT_EQUAL(4, count); + std::cout << count << std::endl; + // We deliberately do it twice: + count = val.count_elements(); + ASSERT_EQUAL(4, count); + std::cout << count << std::endl; + std::vector values(count); + // Next, we check whether we can iterate normally: + size_t index = 0; + for(double x : val) { values[index++] = x; } + ASSERT_EQUAL(index, count); + TEST_SUCCEED(); +} + + +bool json_array_count() { + TEST_START(); + singlestage::parser parser; + auto cars_json = R"( [ 40.1, 39.9, 37.7, 40.4 ] )"_padded; + auto doc = parser.iterate(cars_json); + size_t count = doc.count_elements(); + ASSERT_EQUAL(4, count); + std::cout << count << std::endl; + // We deliberately do it twice: + count = doc.count_elements(); + ASSERT_EQUAL(4, count); + std::cout << count << std::endl; + std::vector values(count); + size_t index = 0; + for(double x : doc) { values[index++] = x; } + ASSERT_EQUAL(index, count); + TEST_SUCCEED(); +} + +bool json_array_count_complex() { + TEST_START(); + singlestage::parser parser; + auto cars_json = R"( { "test":[ { "val1":1, "val2":2 }, { "val1":1, "val2":2 }, { "val1":1, "val2":2 } ] } )"_padded; + auto doc = parser.iterate(cars_json); + auto test_array = doc.find_field("test").get_array(); + size_t count = test_array.count_elements(); + std::cout << "Number of elements: " << count << std::endl; + size_t c = 0; + for(singlestage::object elem : test_array) { + std::cout << simdjson::to_json_string(elem); + c++; + } + std::cout << std::endl; + ASSERT_EQUAL(c, count); + TEST_SUCCEED(); + +} + +bool json_object_count() { + TEST_START(); + auto json = R"( { "test":{ "val1":1, "val2":2 } } )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + size_t count; + ASSERT_SUCCESS(doc.count_fields().get(count)); + ASSERT_EQUAL(count,1); + singlestage::object object; + size_t new_count; + ASSERT_SUCCESS(doc.find_field("test").get_object().get(object)); + ASSERT_SUCCESS(object.count_fields().get(new_count)); + ASSERT_EQUAL(new_count, 2); + TEST_SUCCEED(); +} + +bool using_the_parsed_json_1() { + TEST_START(); + + try { + + singlestage::parser parser; + auto json = R"( { "x": 1, "y": 2 } )"_padded; + auto doc = parser.iterate(json); + double y = doc.find_field("y"); // The cursor is now after the 2 (at }) + double x = doc.find_field("x"); // This fails, because there are no more fields after "y" + + cout << x << ", " << y << endl; + } catch (...) { + TEST_SUCCEED(); + } + TEST_FAIL("expected an exception"); +} + +bool using_the_parsed_json_2() { + TEST_START(); + + singlestage::parser parser; + auto json = R"( { "x": 1, "y": 2 } )"_padded; + auto doc = parser.iterate(json); + double y = doc["y"]; // The cursor is now after the 2 (at }) + double x = doc["x"]; // Success: [] loops back around to find "x" + + cout << x << ", " << y << endl; + + TEST_SUCCEED(); +} + + bool big_integer() { + TEST_START(); + simdjson::singlestage::parser parser; + simdjson::padded_string docdata = R"({"value":12321323213213213213213213213211223})"_padded; + simdjson::singlestage::document doc = parser.iterate(docdata); + simdjson::singlestage::object obj = doc.get_object(); + string_view token = obj["value"].raw_json_token(); + std::cout << token << std::endl; + // token == "12321323213213213213213213213211223" + TEST_SUCCEED(); + } + + bool big_integer_in_string() { + TEST_START(); + simdjson::singlestage::parser parser; + simdjson::padded_string docdata = R"({"value":"12321323213213213213213213213211223"})"_padded; + simdjson::singlestage::document doc = parser.iterate(docdata); + simdjson::singlestage::object obj = doc.get_object(); + string_view token = obj["value"].raw_json_token(); + std::cout << token << std::endl; + // token == "\"12321323213213213213213213213211223\"" + TEST_SUCCEED(); + } +bool using_the_parsed_json_3() { + TEST_START(); + + singlestage::parser parser; + auto cars_json = R"( [ + { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, + { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, + { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] } + ] )"_padded; + + // Iterating through an array of objects + for (singlestage::object car : parser.iterate(cars_json)) { + // Accessing a field by name + cout << "Make/Model: " << std::string_view(car["make"]) << "/" << std::string_view(car["model"]) << endl; + + // Casting a JSON element to an integer + uint64_t year = car["year"]; + cout << "- This car is " << 2020 - year << " years old." << endl; + + // Iterating through an array of floats + double total_tire_pressure = 0; + for (double tire_pressure : car["tire_pressure"]) { + total_tire_pressure += tire_pressure; + } + cout << "- Average tire pressure: " << (total_tire_pressure / 4) << endl; + } + + TEST_SUCCEED(); +} + +bool using_the_parsed_json_3b() { + TEST_START(); + singlestage::parser parser; + auto cars_json = R"( { + "identifier1":{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, + "identifier2":{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, + "identifier3":{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] } + } )"_padded; + + // Iterating through an array of objects + singlestage::document doc = parser.iterate(cars_json); + for (singlestage::field key_car : doc.get_object()) { + // If I need a string_view and/or, I can use key_car.unescaped_key() instead, but + // key_car.key() will be more performant otherwise. + cout << "identifier : " << key_car.key() << std::endl; + // I can now access the subobject: + singlestage::object car = key_car.value(); + // Accessing a field by name + cout << "Make/Model: " << std::string_view(car["make"]) << "/" << std::string_view(car["model"]) << endl; + + // Casting a JSON element to an integer + uint64_t year = car["year"]; + cout << "- This car is " << 2020 - year << "years old." << endl; + + // Iterating through an array of floats + double total_tire_pressure = 0; + for (double tire_pressure : car["tire_pressure"]) { + total_tire_pressure += tire_pressure; + } + cout << "- Average tire pressure: " << (total_tire_pressure / 4) << endl; + } + + TEST_SUCCEED(); +} + +bool using_the_parsed_json_rewind() { + TEST_START(); + + singlestage::parser parser; + auto cars_json = R"( [ + { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, + { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, + { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] } + ] )"_padded; + + auto doc = parser.iterate(cars_json); + size_t count = 0; + for (simdjson_unused singlestage::object car : doc) { + if(car["make"] == "Toyota") { count++; } + } + std::cout << "We have " << count << " Toyota cars.\n"; + doc.rewind(); + for (singlestage::object car : doc) { + cout << "Make/Model: " << std::string_view(car["make"]) << "/" << std::string_view(car["model"]) << endl; + } + TEST_SUCCEED(); +} + + +bool using_the_parsed_json_rewind_array() { + TEST_START(); + + singlestage::parser parser; + auto cars_json = R"( [ + { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, + { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, + { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] } + ] )"_padded; + + auto doc = parser.iterate(cars_json); + singlestage::array arr = doc.get_array(); + size_t count = 0; + for (simdjson_unused singlestage::object car : arr) { + if(car["make"] == "Toyota") { count++; } + } + std::cout << "We have " << count << " Toyota cars.\n"; + arr.reset(); + for (singlestage::object car : arr) { + cout << "Make/Model: " << std::string_view(car["make"]) << "/" << std::string_view(car["model"]) << endl; + } + TEST_SUCCEED(); +} + + +bool using_the_parsed_json_4() { + TEST_START(); + + singlestage::parser parser; + auto points_json = R"( [ + { "12345" : {"x":12.34, "y":56.78, "z": 9998877} }, + { "12545" : {"x":11.44, "y":12.78, "z": 11111111} } + ] )"_padded; + + // Parse and iterate through an array of objects + for (singlestage::object points : parser.iterate(points_json)) { + // Iterating through an object, you iterate through key-value pairs (a 'field'). + for (auto point : points) { + // Get the key corresponding the the field 'point'. + cout << "id: " << std::string_view(point.unescaped_key()) << ": ("; + // Get the value corresponding the the field 'point'. + singlestage::object xyz = point.value(); + cout << xyz["x"].get_double() << ", "; + cout << xyz["y"].get_double() << ", "; + cout << xyz["z"].get_int64() << endl; + } + } + + TEST_SUCCEED(); +} + +bool using_the_parsed_json_5() { + TEST_START(); + + auto abstract_json = R"( + { "str" : { "123" : {"abc" : 3.14 } } } + )"_padded; + singlestage::parser parser; + auto doc = parser.iterate(abstract_json); + cout << doc["str"]["123"]["abc"].get_double() << endl; // Prints 3.14 + + TEST_SUCCEED(); +} + +#endif // SIMDJSON_EXCEPTIONS + + + +bool using_the_parsed_json_no_exceptions() { + TEST_START(); + + singlestage::parser parser; + auto cars_json = R"( [ + { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, + { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, + { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] } + ] )"_padded; + singlestage::document doc; + + // Iterating through an array of objects + auto error = parser.iterate(cars_json).get(doc); + if(error) { std::cerr << error << std::endl; return false; } + singlestage::array cars; + error = doc.get_array().get(cars); + + for (auto car_value : cars) { + singlestage::object car; + error = car_value.get_object().get(car); + if(error) { std::cerr << error << std::endl; return false; } + + // Accessing a field by name + std::string_view make; + std::string_view model; + error = car["make"].get(make); + if(error) { std::cerr << error << std::endl; return false; } + error = car["model"].get(model); + if(error) { std::cerr << error << std::endl; return false; } + + cout << "Make/Model: " << make << "/" << model << endl; + + // Casting a JSON element to an integer + uint64_t year{}; + error = car["year"].get(year); + if(error) { std::cerr << error << std::endl; return false; } + cout << "- This car is " << 2020 - year << " years old." << endl; + + // Iterating through an array of floats + double total_tire_pressure = 0; + singlestage::array pressures; + error = car["tire_pressure"].get_array().get(pressures); + if(error) { std::cerr << error << std::endl; return false; } + for (auto tire_pressure_value : pressures) { + double tire_pressure; + error = tire_pressure_value.get_double().get(tire_pressure); + if(error) { std::cerr << error << std::endl; return false; } + total_tire_pressure += tire_pressure; + } + cout << "- Average tire pressure: " << (total_tire_pressure / 4) << endl; + } + TEST_SUCCEED(); +} +int using_the_parsed_json_6_process() { + auto abstract_json = R"( + { "str" : { "123" : {"abc" : 3.14 } } } + )"_padded; + singlestage::parser parser; + + double value; + auto doc = parser.iterate(abstract_json); + auto error = doc["str"]["123"]["abc"].get(value); + if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; } + cout << value << endl; // Prints 3.14 + + return EXIT_SUCCESS; +} + +bool exception_free_object_loop() { + auto json = R"({"k\u0065y": 1})"_padded; + singlestage::parser parser; + singlestage::document doc; + auto error = parser.iterate(json).get(doc); + if(error) { return false; } + singlestage::object object; + error = doc.get_object().get(object); + if(error) { return false; } + for(auto field : object) { + std::string_view keyv; + error = field.unescaped_key().get(keyv); + if(error) { return false; } + if(keyv == "key") { + uint64_t intvalue; + error = field.value().get(intvalue); + if(error) { return false; } + std::cout << intvalue; + } + } + return true; +} + + +bool exception_free_object_loop_no_escape() { + auto json = R"({"k\u0065y": 1})"_padded; + singlestage::parser parser; + singlestage::document doc; + auto error = parser.iterate(json).get(doc); + if(error) { return false; } + singlestage::object object; + error = doc.get_object().get(object); + if(error) { return false; } + for(auto field : object) { + singlestage::raw_json_string keyv; + error = field.key().get(keyv); + /** + * If you need to escape the keys, you can do + * std::string_view keyv; + * error = field.unescaped_key().get(keyv); + */ + if(error) { return false; } + if(keyv == "key") { + uint64_t intvalue; + error = field.value().get(intvalue); + if(error) { return false; } + std::cout << intvalue; + } + } + return true; +} + +bool using_the_parsed_json_6() { + TEST_START(); + ASSERT_EQUAL(using_the_parsed_json_6_process(), EXIT_SUCCESS); + TEST_SUCCEED(); +} + +const padded_string cars_json = R"( [ + { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, + { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, + { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] } +] )"_padded; + +bool json_pointer_simple() { + TEST_START(); + singlestage::parser parser; + singlestage::document cars; + double x; + ASSERT_SUCCESS(parser.iterate(cars_json).get(cars)); + ASSERT_SUCCESS(cars.at_pointer("/0/tire_pressure/1").get(x)); + ASSERT_EQUAL(x,39.9); + TEST_SUCCEED(); +} + +bool json_pointer_multiple() { + TEST_START(); + singlestage::parser parser; + singlestage::document cars; + size_t size; + ASSERT_SUCCESS(parser.iterate(cars_json).get(cars)); + ASSERT_SUCCESS(cars.count_elements().get(size)); + double expected[] = {39.9, 31, 30}; + for (size_t i = 0; i < size; i++) { + std::string json_pointer = std::string("/") + std::to_string(i) + std::string("/tire_pressure/1"); + double x; + ASSERT_SUCCESS(cars.at_pointer(json_pointer).get(x)); + ASSERT_EQUAL(x,expected[i]); + } + TEST_SUCCEED(); +} + +bool json_pointer_rewind() { + TEST_START(); + auto json = R"( { + "k0": 27, + "k1": [13,26], + "k2": true + } )"_padded; + + singlestage::parser parser; + singlestage::document doc; + uint64_t i; + bool b; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ASSERT_SUCCESS(doc.at_pointer("/k1/1").get(i)); + ASSERT_EQUAL(i,26); + ASSERT_SUCCESS(doc.at_pointer("/k2").get(b)); + ASSERT_EQUAL(b,true); + doc.rewind(); // Need to manually rewind to be able to use find_field properly from start of document + ASSERT_SUCCESS(doc.find_field("k0").get(i)); + ASSERT_EQUAL(i,27); + TEST_SUCCEED(); +} + +bool iterate_many_example() { + TEST_START(); + auto json = R"([1,2,3] {"1":1,"2":3,"4":4} [1,2,3] )"_padded; + simdjson::singlestage::parser parser; + simdjson::singlestage::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(json).get(stream)); + auto i = stream.begin(); + size_t count{0}; + size_t expected_indexes[3] = {0,9,29}; + std::string_view expected_doc[3] = {"[1,2,3]", R"({"1":1,"2":3,"4":4})", "[1,2,3]"}; + for(; i != stream.end(); ++i) { + auto doc = *i; + ASSERT_SUCCESS(doc.type()); + ASSERT_SUCCESS(i.error()); + ASSERT_EQUAL(i.current_index(),expected_indexes[count]); + ASSERT_EQUAL(i.source(),expected_doc[count]); + count++; + } + TEST_SUCCEED(); +} + +std::string my_string(singlestage::document& doc) { + std::stringstream ss; + ss << doc; + return ss.str(); +} + +bool iterate_many_truncated_example() { + TEST_START(); + auto json = R"([1,2,3] {"1":1,"2":3,"4":4} {"key":"intentionally unclosed string )"_padded; + simdjson::singlestage::parser parser; + simdjson::singlestage::document_stream stream; + ASSERT_SUCCESS( parser.iterate_many(json,json.size()).get(stream) ); + std::string_view expected[2] = {"[1,2,3]", R"({"1":1,"2":3,"4":4})"}; + size_t count{0}; + for(auto i = stream.begin(); i != stream.end(); ++i) { + ASSERT_EQUAL(i.source(),expected[count++]); + } + size_t truncated = stream.truncated_bytes(); + ASSERT_EQUAL(truncated,39); + TEST_SUCCEED(); +} + +bool ndjson_basics_example() { + TEST_START(); + auto json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )"_padded; + singlestage::parser parser; + singlestage::document_stream docs; + ASSERT_SUCCESS( parser.iterate_many(json).get(docs) ); + size_t count{0}; + int64_t expected[3] = {1,2,3}; + for (auto doc : docs) { + int64_t actual{}; + ASSERT_SUCCESS( doc["foo"].get(actual) ); + ASSERT_EQUAL( actual, expected[count++] ); + } + TEST_SUCCEED(); +} +bool stream_capacity_example() { + auto json = R"([1,2,3,4,5] [1,2,3,4,5] [1,2,3,4,5] [1,2,3,4,5] [1,2,3,4,5] [1,2,3,4,5] [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100])"_padded; + singlestage::parser parser; + singlestage::document_stream stream; + size_t counter{0}; + auto error = parser.iterate_many(json, 50).get(stream); + if( error ) { /* handle the error */ } + for (auto doc: stream) { + if(counter < 6) { + int64_t val{}; + error = doc.at_pointer("/4").get(val); + if( error ) { /* handle the error */ } + std::cout << "5 = " << val << std::endl; + } else { + singlestage::value val; + error = doc.at_pointer("/4").get(val); + // error == simdjson::CAPACITY + if(error) { + std::cerr << error << std::endl; + // We left 293 bytes unprocessed at the tail end of the input. + std::cout << " unprocessed bytes at the end: " << stream.truncated_bytes() << std::endl; + break; + } + } + counter++; + } + return true; +} + + + +bool simple_error_example() { + singlestage::parser parser; + auto json = R"({"bad number":3.14.1 })"_padded; + singlestage::document doc; + if( parser.iterate(json).get(doc) != SUCCESS ) { return false; } + double x; + auto error = doc["bad number"].get_double().get(x); + // returns "simdjson::NUMBER_ERROR" + if(error != SUCCESS) { + std::cout << error << std::endl; + return false; + } + std::cout << "Got " << x << std::endl; + return true; +} + + +#if SIMDJSON_EXCEPTIONS + bool raw_string() { + TEST_START(); + auto json = R"( {"name": "Jack The Ripper \u0033"} )"_padded; + // We create a buffer large enough to store all strings we need: + std::unique_ptr buffer(new uint8_t[json.size() + simdjson::SIMDJSON_PADDING]); + uint8_t * ptr = buffer.get(); + singlestage::parser parser; + singlestage::document doc = parser.iterate(json); + // We store our strings as 'string_view' instances in a vector: + std::vector mystrings; + for (auto key_value : doc.get_object()) { + std::string_view keysv = parser.unescape(key_value.key(), ptr);// writes 'name' + mystrings.push_back(keysv); + std::string_view valuesv = parser.unescape(key_value.value().get_raw_json_string(), ptr); + // writes 'Jack The Ripper 3', escaping the \u0033 + mystrings.push_back(valuesv); + } + ASSERT_EQUAL(mystrings[0],"name"); + ASSERT_EQUAL(mystrings[1],"Jack The Ripper 3"); + TEST_SUCCEED(); + } + + + bool simple_error_example_except() { + TEST_START(); + singlestage::parser parser; + auto json = R"({"bad number":3.14.1 })"_padded; + try { + singlestage::document doc = parser.iterate(json); + double x = doc["bad number"].get_double(); + std::cout << "Got " << x << std::endl; + TEST_SUCCEED(); + } catch(simdjson_error& e) { + // e.error() == NUMBER_ERROR + std::cout << e.error() << std::endl; + TEST_FAIL("I did not expect an exception"); + } + } + + int64_t current_location_tape_error_with_except() { + TEST_START(); + auto broken_json = R"( {"double": 13.06, false, "integer": -343} )"_padded; + singlestage::parser parser; + singlestage::document doc; + try { + doc = parser.iterate(broken_json); + return int64_t(doc["integer"]); + } catch(simdjson_error& err) { + std::cout << err.error() << std::endl; + std::cout << doc.current_location() << std::endl; + TEST_SUCCEED(); + } + TEST_FAIL("I expected an exception!"); + } + +#endif + +int load_example() { + TEST_START(); + simdjson::singlestage::parser parser; + simdjson::singlestage::document tweets; + padded_string json; + auto error = padded_string::load("twitter.json").get(json); + if(error) { std::cerr << error << std::endl; return EXIT_FAILURE; } + error = parser.iterate(json).get(tweets); + if(error) { std::cerr << error << std::endl; return EXIT_FAILURE; } + uint64_t identifier; + error = tweets["statuses"].at(0)["id"].get(identifier); + if(error) { std::cerr << error << std::endl; return EXIT_FAILURE; } + std::cout << identifier << std::endl; + return EXIT_SUCCESS; +} + + +int example_1() { + TEST_START(); + simdjson::singlestage::parser parser; + //auto error = padded_string::load("twitter.json").get(json); + // if(error) { std::cerr << error << std::endl; return EXIT_FAILURE; } + padded_string json = R"( { + "statuses": [ + { + "id": 505874924095815700 + }, + { + "id": 505874922023837700 + } + ], + "search_metadata": { + "count": 100 + } +} )"_padded; + + simdjson::singlestage::document tweets; + auto error = parser.iterate(json).get(tweets); + if( error ) { return EXIT_FAILURE; } + simdjson::singlestage::value res; + error = tweets["search_metadata"]["count"].get(res); + if (error != SUCCESS) { + std::cerr << "could not access keys" << std::endl; + return EXIT_FAILURE; + } + std::cout << res << " results." << std::endl; + return true; +} +#if SIMDJSON_EXCEPTIONS +int load_example_except() { + TEST_START(); + simdjson::singlestage::parser parser; + padded_string json = padded_string::load("twitter.json"); + simdjson::singlestage::document tweets = parser.iterate(json); + uint64_t identifier = tweets["statuses"].at(0)["id"]; + std::cout << identifier << std::endl; + return EXIT_SUCCESS; +} +int load_example_except_morecomplete(void) { + TEST_START(); + simdjson::singlestage::parser parser; + simdjson::padded_string json_string; + simdjson::singlestage::document doc; + try { + json_string = padded_string::load("twitter.json"); + doc = parser.iterate(json_string); + uint64_t identifier = doc["statuses"].at(0)["id"]; + std::cout << identifier << std::endl; + } catch (simdjson::simdjson_error &error) { + std::cerr << "JSON error: " << error.what() << " near " + << doc.current_location() << " in " << json_string << std::endl; + } + return EXIT_SUCCESS; +} + +bool allow_comma_separated_example() { + TEST_START(); + auto json = R"( 1, 2, 3, 4, "a", "b", "c", {"hello": "world"} , [1, 2, 3])"_padded; + singlestage::parser parser; + singlestage::document_stream doc_stream; + // We pass '32' as the batch size, but it is a bogus parameter because, since + // we pass 'true' to the allow_comma parameter, the batch size will be set to at least + // the document size. + auto error = parser.iterate_many(json, 32, true).get(doc_stream); + if(error) { std::cerr << error << std::endl; return false; } + for (auto doc : doc_stream) { + std::cout << doc.type() << std::endl; + } + TEST_SUCCEED(); +} +#endif +bool test_load_example() { + TEST_START(); + simdjson::singlestage::parser parser; + simdjson::singlestage::document tweets; + padded_string json = R"( {"statuses":[{"id":1234}]} )"_padded; + auto error = parser.iterate(json).get(tweets); + if(error) { std::cerr << error << std::endl; return false; } + uint64_t identifier; + error = tweets["statuses"].at(0)["id"].get(identifier); + if(error) { std::cerr << error << std::endl; return false; } + std::cout << identifier << std::endl; + return identifier == 1234; +} + +bool current_location_tape_error() { + TEST_START(); + auto broken_json = R"( {"double": 13.06, false, "integer": -343} )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(broken_json).get(doc)); + const char * ptr; + int64_t i; + ASSERT_ERROR(doc["integer"].get_int64().get(i), TAPE_ERROR); + ASSERT_SUCCESS(doc.current_location().get(ptr)); + std::string expected = "false, \"integer\": -343} "; + ASSERT_EQUAL(std::string(ptr,expected.size()), expected); + TEST_SUCCEED(); +} + +bool current_location_user_error() { + TEST_START(); + auto json = R"( [1,2,3] )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + const char * ptr; + int64_t i; + ASSERT_ERROR(doc["integer"].get_int64().get(i), INCORRECT_TYPE); + ASSERT_SUCCESS(doc.current_location().get(ptr)); + std::string expected = "[1,2,3] "; + ASSERT_EQUAL(std::string(ptr, expected.size()), expected); + TEST_SUCCEED(); +} + +bool current_location_out_of_bounds() { + TEST_START(); + auto json = R"( [1,2,3] )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + uint64_t expected[3] = {1, 2, 3}; + size_t count{0}; + for (auto val : doc) { + uint64_t i; + ASSERT_SUCCESS(val.get_uint64().get(i)); + ASSERT_EQUAL(i, expected[count++]); + } + ASSERT_EQUAL(count, 3); + ASSERT_ERROR(doc.current_location(), OUT_OF_BOUNDS); + TEST_SUCCEED(); +} + +bool current_location_no_error() { + TEST_START(); + auto json = R"( [[1,2,3], -23.4, {"key": "value"}, true] )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + const char * ptr; + for (auto val : doc) { + singlestage::object obj; + auto error = val.get_object().get(obj); + if (!error) { + ASSERT_SUCCESS(doc.current_location().get(ptr)); + std::string expected = "\"key\": \"value\"}, true] "; + ASSERT_EQUAL(std::string(ptr, expected.size()), expected); + } + } + TEST_SUCCEED(); +} + + + +struct ZuluBBox { + double xmin; + double ymin; + double width; + double height; + + void print() { + std::cout << xmin << ", " << ymin << ", " << width << ", " << height + << std::endl; + } +}; +#if SIMDJSON_EXCEPTIONS + +bool example1956() { + + auto json = R"+( { + "ZuluROI": { + "ZuluBBox": { + "xmin": 0, + "ymin": 0, + "width": 1, + "height": 1 + }, + "SubObjects": [ + { + "ZuluDetection": { + "label": "car", + "class_id": 3, + "confidence": 0.7587034106254578, + "ZuluBBox": { + "xmin": 0.3843536376953125, + "ymin": 0.4532909393310547, + "width": 0.09115534275770187, + "height": 0.04127710685133934 + }, + "SubObjects": [] + } + }, + { + "ZuluDetection": { + "label": "car", + "class_id": 3, + "confidence": 0.6718865633010864, + "ZuluBBox": { + "xmin": 0.7500002980232239, + "ymin": 0.5212296843528748, + "width": 0.07592231780290604, + "height": 0.038947589695453644 + }, + "SubObjects": [] + } + }, + { + "ZuluDetection": { + "label": "car", + "class_id": 3, + "confidence": 0.5806200504302979, + "ZuluBBox": { + "xmin": 0.9025363922119141, + "ymin": 0.5925348401069641, + "width": 0.05478987470269203, + "height": 0.046337299048900604 + }, + "SubObjects": [] + } + } + ] + }, + "timestamp (ms)": 1677085594421, + "buffer_offset": 35673 +} )+"_padded; + singlestage::parser parser; + singlestage::document doc = parser.iterate(json); + singlestage::object root_object = doc.get_object(); + singlestage::object roi_object = root_object["ZuluROI"]; + + singlestage::object box_roi_object = roi_object["ZuluBBox"]; + ZuluBBox box = { + double(box_roi_object["xmin"]), double(box_roi_object["ymin"]), + double(box_roi_object["width"]), double(box_roi_object["height"])}; + box.print(); + + for (singlestage::object value : roi_object["SubObjects"]) { + singlestage::object detect = value["ZuluDetection"]; + std::cout << detect["label"].get_string() << std::endl; + std::cout << detect["class_id"].get_uint64() << std::endl; + std::cout << detect["confidence"].get_double() << std::endl; + + singlestage::object vbox_roi_object = detect["ZuluBBox"]; + ZuluBBox vbox = { + double(vbox_roi_object["xmin"]), double(vbox_roi_object["ymin"]), + double(vbox_roi_object["width"]), double(vbox_roi_object["height"])}; + vbox.print(); + } + + std::cout << root_object["timestamp (ms)"].get_uint64() << std::endl; + std::cout << root_object["buffer_offset"].get_uint64() << std::endl; + return true; +} + + +bool example1958() { + auto json = R"+( { + "5f08a730b280e54fd1e75a7046b93fdc": { + "file": "/DEMOS/0-9/10_Orbyte.sid", + "len": [ + "1:17" + ], + "loud": [ + "-22.8" + ], + "name": "10 Orbyte", + "author": "Michael Becker (Premium)", + "release": "2014 Tristar & Red Sector Inc.", + "bits": 20 + }, + "2727236ead44a62f0c6e01f6dd4dc484": { + "file": "/DEMOS/0-9/12345.sid", + "len": [ + "0:56" + ], + "loud": [ + "-33.3" + ], + "name": "12345", + "author": "Beal", + "release": "1988 Beal", + "bits": 20 + }, + "7ea765fce6c0f92570b18adc7bf52f54": { + "file": "/DEMOS/0-9/128_Byte_Blues_BASIC.sid", + "len": [ + "0:18" + ], + "loud": [ + "-27.1" + ], + "name": "128 Byte Blues", + "author": "Leonard J. Paul (Freaky DNA)", + "release": "2005 Freaky DNA", + "bits": 62 + } +} )+"_padded; + singlestage::parser parser; + singlestage::document doc = parser.iterate(json); + singlestage::object root_object = doc.get_object(); + for(auto key_value : root_object) { + // could get std::string_view with 'unescaped_key()': + std::cout << "key: " << key_value.key() << std::endl; + singlestage::object obj = key_value.value(); + + std::cout << "file: " << std::string_view(obj["file"]) << std::endl; + + std::cout << "len: "; + for(std::string_view values : obj["len"]) { + std::cout << values << std::endl; + } + std::cout << std::endl; + + std::cout << "loud: "; + for(std::string_view values : obj["loud"]) { + std::cout << values << std::endl; + } + std::cout << std::endl; + + std::cout << "name: " << std::string_view(obj["name"]) << std::endl; + std::cout << "author: " << std::string_view(obj["author"]) << std::endl; + std::cout << "release: " << std::string_view(obj["release"]) << std::endl; + std::cout << "bits: " << uint64_t(obj["bits"]) << std::endl; + } + return true; +} +#endif +bool run() { + return true +#if SIMDJSON_EXCEPTIONS + && gen_raw1() && gen_raw2() && gen_raw3() + && at_end() + && example1956() && example1958() + && allow_comma_separated_example() +// && basics_1() // Fails because twitter.json isn't in current directory. Compile test only. + && basics_treewalk() + && basics_treewalk_breakline() + && json_value_with_array_count() + && json_array_with_array_count() + && json_array_count_complex() + && json_array_count() + && json_object_count() + && using_the_parsed_json_rewind() + && using_the_parsed_json_rewind_array() + && basics_2() + && using_the_parsed_json_1() + && using_the_parsed_json_2() + && big_integer() + && big_integer_in_string() + && using_the_parsed_json_3() + && using_the_parsed_json_3b() + && using_the_parsed_json_4() + && using_the_parsed_json_5() +#endif + && using_the_parsed_json_6() + && json_pointer_simple() + && json_pointer_multiple() + && json_pointer_rewind() + && iterate_many_example() + && iterate_many_truncated_example() + && ndjson_basics_example() + && stream_capacity_example() + && test_load_example() + && example_1() + && using_the_parsed_json_no_exceptions() + && current_location_tape_error() + && current_location_user_error() + && current_location_out_of_bounds() + && current_location_no_error() + #if SIMDJSON_EXCEPTIONS + && raw_string() + && number_tests() + && current_location_tape_error_with_except() + #endif + ; +} + +int main(int argc, char *argv[]) { + return test_main(argc, argv, run); +} diff --git a/tests/singlestage/singlestage_scalar_tests.cpp b/tests/singlestage/singlestage_scalar_tests.cpp new file mode 100644 index 000000000..1c836ceee --- /dev/null +++ b/tests/singlestage/singlestage_scalar_tests.cpp @@ -0,0 +1,402 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace scalar_tests { + using namespace std; + using simdjson::singlestage::json_type; + + template json_type expected_json_type(); + template<> json_type expected_json_type() { return json_type::string; } + template<> json_type expected_json_type() { return json_type::number; } + template<> json_type expected_json_type() { return json_type::number; } + template<> json_type expected_json_type() { return json_type::number; } + template<> json_type expected_json_type() { return json_type::boolean; } + + template + bool test_scalar_value(const padded_string &json, const T &expected, bool test_twice=true) { + std::cout << "- JSON: " << json << endl; + SUBTEST( "simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + T actual{}; + ASSERT_RESULT( doc_result.type(), expected_json_type() ); + ASSERT_SUCCESS( doc_result.get(actual) ); + ASSERT_EQUAL( actual, expected ); + // Test it twice (scalars can be retrieved more than once) + if (test_twice) { + ASSERT_SUCCESS( doc_result.get(actual) ); + ASSERT_EQUAL( actual, expected ); + ASSERT_RESULT( doc_result.type(), expected_json_type() ); + } + return true; + })); + SUBTEST( "document", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::document doc; + ASSERT_SUCCESS( std::move(doc_result).get(doc) ); + T actual{}; + ASSERT_RESULT( doc.type(), expected_json_type() ); + ASSERT_SUCCESS( doc.get(actual) ); + ASSERT_EQUAL( actual, expected ); + // Test it twice (scalars can be retrieved more than once) + if (test_twice) { + ASSERT_SUCCESS( doc.get(actual) ); + ASSERT_EQUAL( actual, expected ); + ASSERT_RESULT( doc.type(), expected_json_type() ); + } + return true; + })); + + { + padded_string whitespace_json = std::string(json) + " "; + std::cout << "- JSON: " << whitespace_json << endl; + SUBTEST( "simdjson_result", test_singlestage_doc(whitespace_json, [&](auto doc_result) { + T actual{}; + ASSERT_RESULT( doc_result.type(), expected_json_type() ); + ASSERT_SUCCESS( doc_result.get(actual) ); + ASSERT_EQUAL( actual, expected ); + // Test it twice (scalars can be retrieved more than once) + if (test_twice) { + ASSERT_SUCCESS( doc_result.get(actual) ); + ASSERT_EQUAL( actual, expected ); + ASSERT_RESULT( doc_result.type(), expected_json_type() ); + } + return true; + })); + SUBTEST( "document", test_singlestage_doc(whitespace_json, [&](auto doc_result) { + singlestage::document doc; + ASSERT_SUCCESS( std::move(doc_result).get(doc) ); + T actual{}; + ASSERT_RESULT( doc.type(), expected_json_type() ); + ASSERT_SUCCESS( doc.get(actual) ); + ASSERT_EQUAL( actual, expected ); + // Test it twice (scalars can be retrieved more than once) + if (test_twice) { + ASSERT_SUCCESS( doc.get(actual) ); + ASSERT_EQUAL( actual, expected ); + ASSERT_RESULT( doc.type(), expected_json_type() ); + } + return true; + })); + } + + { + padded_string array_json = std::string("[") + std::string(json) + "]"; + std::cout << "- JSON: " << array_json << endl; + SUBTEST( "simdjson_result", test_singlestage_doc(array_json, [&](auto doc_result) { + int count = 0; + for (simdjson_result val_result : doc_result) { + T actual{}; + ASSERT_RESULT( val_result.type(), expected_json_type() ); + ASSERT_SUCCESS( val_result.get(actual) ); + ASSERT_EQUAL(actual, expected); + // Test it twice (scalars can be retrieved more than once) + if (test_twice) { + ASSERT_SUCCESS( val_result.get(actual) ); + ASSERT_EQUAL(actual, expected); + ASSERT_RESULT( val_result.type(), expected_json_type() ); + } + count++; + } + ASSERT_EQUAL(count, 1); + return true; + })); + SUBTEST( "value", test_singlestage_doc(array_json, [&](auto doc_result) { + int count = 0; + for (simdjson_result val_result : doc_result) { + singlestage::value val; + ASSERT_SUCCESS( val_result.get(val) ); + T actual{}; + ASSERT_RESULT( val.type(), expected_json_type() ); + ASSERT_SUCCESS( val.get(actual) ); + ASSERT_EQUAL(actual, expected); + // Test it twice (scalars can be retrieved more than once) + if (test_twice) { + ASSERT_SUCCESS( val.get(actual) ); + ASSERT_EQUAL(actual, expected); + ASSERT_RESULT( val.type(), expected_json_type() ); + } + count++; + } + ASSERT_EQUAL(count, 1); + return true; + })); + } + + { + padded_string whitespace_array_json = std::string("[") + std::string(json) + " ]"; + std::cout << "- JSON: " << whitespace_array_json << endl; + + SUBTEST( "simdjson_result", test_singlestage_doc(whitespace_array_json, [&](auto doc_result) { + int count = 0; + for (simdjson_result val_result : doc_result) { + T actual{}; + ASSERT_RESULT( val_result.type(), expected_json_type() ); + ASSERT_SUCCESS( val_result.get(actual) ); + ASSERT_EQUAL(actual, expected); + // Test it twice (scalars can be retrieved more than once) + if (test_twice) { + ASSERT_SUCCESS( val_result.get(actual) ); + ASSERT_EQUAL(actual, expected); + ASSERT_RESULT( val_result.type(), expected_json_type() ); + } + count++; + } + ASSERT_EQUAL(count, 1); + return true; + })); + + SUBTEST( "value", test_singlestage_doc(whitespace_array_json, [&](auto doc_result) { + int count = 0; + for (simdjson_result val_result : doc_result) { + singlestage::value val; + ASSERT_SUCCESS( val_result.get(val) ); + T actual{}; + ASSERT_RESULT( val.type(), expected_json_type() ); + ASSERT_SUCCESS( val.get(actual) ); + ASSERT_EQUAL(actual, expected); + // Test it twice (scalars can be retrieved more than once) + if (test_twice) { + ASSERT_SUCCESS( val.get(actual) ); + ASSERT_EQUAL(actual, expected); + ASSERT_RESULT( val_result.type(), expected_json_type() ); + } + count++; + } + ASSERT_EQUAL(count, 1); + return true; + })); + } + + TEST_SUCCEED(); + } + + bool string_value() { + TEST_START(); + // We can't retrieve a small string twice because it will blow out the string buffer + if (!test_scalar_value(R"("hi")"_padded, std::string_view("hi"), false)) { return false; } + // ... unless the document is big enough to have a big string buffer :) + if (!test_scalar_value(R"("hi" )"_padded, std::string_view("hi"))) { return false; } + TEST_SUCCEED(); + } + + bool numeric_values() { + TEST_START(); + if (!test_scalar_value ("0"_padded, 0)) { return false; } + if (!test_scalar_value("0"_padded, 0)) { return false; } + if (!test_scalar_value ("0"_padded, 0)) { return false; } + if (!test_scalar_value ("1"_padded, 1)) { return false; } + if (!test_scalar_value("1"_padded, 1)) { return false; } + if (!test_scalar_value ("1"_padded, 1)) { return false; } + if (!test_scalar_value ("-1"_padded, -1)) { return false; } + if (!test_scalar_value ("-1"_padded, -1)) { return false; } + if (!test_scalar_value ("1.1"_padded, 1.1)) { return false; } + TEST_SUCCEED(); + } + + bool boolean_values() { + TEST_START(); + if (!test_scalar_value ("true"_padded, true)) { return false; } + if (!test_scalar_value ("false"_padded, false)) { return false; } + TEST_SUCCEED(); + } + + bool null_value() { + TEST_START(); + auto json = "null"_padded; + SUBTEST("singlestage::document", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::document doc; + ASSERT_SUCCESS( std::move(doc_result).get(doc) ); + bool is_null_value; + ASSERT_SUCCESS( doc.is_null().get(is_null_value) ); + ASSERT_EQUAL( is_null_value, true ); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + bool is_null_value; + ASSERT_SUCCESS( doc_result.is_null().get(is_null_value) ); + ASSERT_EQUAL( is_null_value, true ); + return true; + })); + json = "[null]"_padded; + SUBTEST("singlestage::value", test_singlestage_doc(json, [&](auto doc_result) { + int count = 0; + for (auto value_result : doc_result) { + singlestage::value value; + ASSERT_SUCCESS( value_result.get(value) ); + bool is_null_value; + ASSERT_SUCCESS( value.is_null().get(is_null_value) ); + ASSERT_EQUAL( is_null_value, true ); + count++; + } + ASSERT_EQUAL( count, 1 ); + return true; + })); + SUBTEST("simdjson_result", test_singlestage_doc(json, [&](auto doc_result) { + int count = 0; + for (auto value_result : doc_result) { + bool is_null_value; + ASSERT_SUCCESS( value_result.is_null().get(is_null_value) ); + ASSERT_EQUAL( is_null_value, true ); + count++; + } + ASSERT_EQUAL( count, 1 ); + return true; + })); + return true; + } + +#if SIMDJSON_EXCEPTIONS + + template + bool test_scalar_value_exception(const padded_string &json, const T &expected) { + std::cout << "- JSON: " << json << endl; + SUBTEST( "document", test_singlestage_doc(json, [&](auto doc_result) { + singlestage::document doc; + ASSERT_SUCCESS( std::move(doc_result).get(doc) ); + ASSERT_EQUAL( expected, T(doc) ); + return true; + })); + padded_string array_json = std::string("[") + std::string(json) + "]"; + std::cout << "- JSON: " << array_json << endl; + SUBTEST( "value", test_singlestage_doc(array_json, [&](auto doc_result) { + int count = 0; + for (T actual : doc_result) { + ASSERT_EQUAL( actual, expected ); + count++; + } + ASSERT_EQUAL(count, 1); + return true; + })); + TEST_SUCCEED(); + } + bool string_value_exception() { + TEST_START(); + return test_scalar_value_exception(R"("hi")"_padded, std::string_view("hi")); + } + + bool numeric_values_exception() { + TEST_START(); + if (!test_scalar_value_exception ("0"_padded, 0)) { return false; } + if (!test_scalar_value_exception("0"_padded, 0)) { return false; } + if (!test_scalar_value_exception ("0"_padded, 0)) { return false; } + if (!test_scalar_value_exception ("1"_padded, 1)) { return false; } + if (!test_scalar_value_exception("1"_padded, 1)) { return false; } + if (!test_scalar_value_exception ("1"_padded, 1)) { return false; } + if (!test_scalar_value_exception ("-1"_padded, -1)) { return false; } + if (!test_scalar_value_exception ("-1"_padded, -1)) { return false; } + if (!test_scalar_value_exception ("1.1"_padded, 1.1)) { return false; } + TEST_SUCCEED(); + } + + bool boolean_values_exception() { + TEST_START(); + if (!test_scalar_value_exception ("true"_padded, true)) { return false; } + if (!test_scalar_value_exception ("false"_padded, false)) { return false; } + TEST_SUCCEED(); + } + +#endif // SIMDJSON_EXCEPTIONS + + bool string_with_trailing() { + TEST_START(); + auto json = R"( "a string" badstuff )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + std::string_view view; + ASSERT_ERROR(doc.get_string().get(view), TRAILING_CONTENT); + TEST_SUCCEED(); + } + + bool uint64_with_trailing() { + TEST_START(); + auto json = R"( 133 badstuff )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + uint64_t x; + ASSERT_ERROR(doc.get_uint64().get(x), TRAILING_CONTENT); + TEST_SUCCEED(); + } + + bool int64_with_trailing() { + TEST_START(); + auto json = R"( 133 badstuff )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + int64_t x; + ASSERT_ERROR(doc.get_int64().get(x), TRAILING_CONTENT); + TEST_SUCCEED(); + } + + bool double_with_trailing() { + TEST_START(); + auto json = R"( 133 badstuff )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + double x; + ASSERT_ERROR(doc.get_double().get(x), TRAILING_CONTENT); + TEST_SUCCEED(); + } + + + bool bool_with_trailing() { + TEST_START(); + auto json = R"( true badstuff )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + bool x; + ASSERT_ERROR(doc.get_bool().get(x), TRAILING_CONTENT); + TEST_SUCCEED(); + } + + bool null_with_trailing() { + TEST_START(); + auto json = R"( null badstuff )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ASSERT_ERROR(doc.is_null(), TRAILING_CONTENT); + TEST_SUCCEED(); + } + + bool nully() { + TEST_START(); + auto json = R"( nully )"_padded; + singlestage::parser parser; + singlestage::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + bool x; + ASSERT_SUCCESS(doc.is_null().get(x)); + ASSERT_TRUE(!x); + TEST_SUCCEED(); + } + + bool run() { + return + nully() && + string_with_trailing() && + uint64_with_trailing() && + int64_with_trailing() && + bool_with_trailing() && + null_with_trailing() && + string_value() && + numeric_values() && + boolean_values() && + null_value() && +#if SIMDJSON_EXCEPTIONS + string_value_exception() && + numeric_values_exception() && + boolean_values_exception() && +#endif // SIMDJSON_EXCEPTIONS + true; + } + +} // namespace scalar_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, scalar_tests::run); +} diff --git a/tests/singlestage/singlestage_tostring_tests.cpp b/tests/singlestage/singlestage_tostring_tests.cpp new file mode 100644 index 000000000..948bde540 --- /dev/null +++ b/tests/singlestage/singlestage_tostring_tests.cpp @@ -0,0 +1,465 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "simdjson.h" + + +/** + * What follows is merely a compilation test to verify + * https://github.com/simdjson/simdjson/issues/1768 + * It deliberately resides before any namespace declaration + * within this file. + */ +namespace issue1768 { + template void print(const T &a) { std::cout << a; } +} // namespace NA + +void issue1768_test() { + simdjson::singlestage::object obj; + issue1768::print(obj); +} +/** + * End of 1768 compilation check. + */ + +using namespace simdjson; + +#include "test_singlestage.h" +namespace tostring_tests { +const char *test_files[] = { + TWITTER_JSON, TWITTER_TIMELINE_JSON, REPEAT_JSON, CANADA_JSON, + MESH_JSON, APACHE_JSON, GSOC_JSON}; + +#if SIMDJSON_EXCEPTIONS +bool issue1607() { + TEST_START(); + auto silly_json = R"( { "test": "result" } )"_padded; + singlestage::parser parser; + singlestage::document doc = parser.iterate(silly_json); + std::string_view expected = R"("result")"; + std::string_view result = simdjson::to_json_string(doc["test"]); + std::cout << "'"<< result << "'" << std::endl; + ASSERT_EQUAL(result, expected); + TEST_SUCCEED(); +} + +bool minify_demo() { + TEST_START(); + singlestage::parser parser; + auto silly_json = R"( { "test": "result" } )"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(silly_json).get(doc) ); + std::cout << simdjson::to_json_string(doc["test"]) << std::endl; + TEST_SUCCEED(); +} + +bool minify_demo2() { + TEST_START(); + singlestage::parser parser; + auto silly_json = R"( { "test": "result" } )"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(silly_json).get(doc) ); + std::cout << std::string_view(doc["test"]) << std::endl; + TEST_SUCCEED(); +} +bool car_example() { + TEST_START(); + auto cars_json = R"( [ + { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, + { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, + { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] } + ] )"_padded; + std::vector arrays; + // We are going to collect string_view instances which point inside the `cars_json` string + // and are therefore valid as long as `cars_json` remains in scope. + { + singlestage::parser parser; + for (singlestage::object car : parser.iterate(cars_json)) { + if(uint64_t(car["year"]) > 2000) { // Pick the recent cars only! + arrays.push_back(simdjson::to_json_string(car["tire_pressure"])); + } + } + } + // We can now convert to a JSON string: + std::ostringstream oss; + oss << "["; + for(size_t i = 0; i < arrays.size(); i++) { + if(i>0) { oss << ","; } + oss << arrays[i]; + } + oss << "]"; + auto json_string = oss.str(); + ASSERT_EQUAL(json_string, "[[ 40.1, 39.9, 37.7, 40.4 ],[ 30.1, 31.0, 28.6, 28.7 ]]"); + TEST_SUCCEED(); +} + + +/** + * The general idea of these tests if that if you take a JSON file, + * load it, then convert it into a string, then parse that, and + * convert it again into a second string, then the two strings should + * be identifical. If not, then something was lost or added in the + * process. + */ + +bool load_to_string(const char *filename) { + simdjson::singlestage::parser parser; + std::cout << "Loading " << filename << std::endl; + simdjson::padded_string docdata; + auto error = simdjson::padded_string::load(filename).get(docdata); + if (error) { + std::cerr << "could not load " << filename << " got " << error << std::endl; + return false; + } + std::cout << "file loaded: " << docdata.size() << " bytes." << std::endl; + simdjson::singlestage::document doc; + auto silly_json = R"( { "test": "result" } )"_padded; + + error = parser.iterate(silly_json).get(doc); + if (error) { + std::cerr << error << std::endl; + return false; + } + std::cout << "serializing once." << std::endl; + std::string_view serial1 = simdjson::to_json_string(doc); + error = parser.iterate(serial1, serial1.size() + simdjson::SIMDJSON_PADDING).get(doc); + if (error) { + std::cerr << error << std::endl; + return false; + } + std::cout << "serializing twice." << std::endl; + std::string_view serial2 = simdjson::to_json_string(doc); + bool match = (serial1 == serial2); + if (match) { + std::cout << "Parsing to_string and calling to_string again results in the " + "same content. " + << "Got " << serial1.size() << " bytes." << std::endl; + } + return match; +} + +bool minify_test() { + TEST_START(); + for (size_t i = 0; i < sizeof(test_files) / sizeof(test_files[0]); i++) { + bool ok = load_to_string(test_files[i]); + if (!ok) { + return false; + } + } + TEST_SUCCEED(); +} + +#endif // SIMDJSON_EXCEPTIONS + +bool load_to_string_exceptionless(const char *filename) { + simdjson::singlestage::parser parser; + std::cout << "Loading " << filename << std::endl; + simdjson::padded_string docdata; + auto error = simdjson::padded_string::load(filename).get(docdata); + if (error) { + std::cerr << "could not load " << filename << " got " << error << std::endl; + return false; + } + std::cout << "file loaded: " << docdata.size() << " bytes." << std::endl; + simdjson::singlestage::document doc; + error = parser.iterate(docdata).get(doc); + if (error) { + std::cerr << error << std::endl; + return false; + } + std::cout << "serializing once." << std::endl; + std::string_view serial1; + error = simdjson::to_json_string(doc).get(serial1); + if (error) { + std::cerr << error << std::endl; + return false; + } + error = parser.iterate(serial1, serial1.size() + simdjson::SIMDJSON_PADDING).get(doc); + if (error) { + std::cerr << error << std::endl; + return false; + } + std::cout << "serializing twice." << std::endl; + std::string_view serial2; + error = simdjson::to_json_string(doc).get(serial2); + if (error) { + std::cerr << error << std::endl; + return false; + } + bool match = (serial1 == serial2); + if (match) { + std::cout << "Parsing to_string and calling to_string again results in the " + "same content. " + << "Got " << serial1.size() << " bytes." << std::endl; + } + return match; +} + +bool minify_exceptionless_test() { + TEST_START(); + for (size_t i = 0; i < sizeof(test_files) / sizeof(test_files[0]); i++) { + bool ok = load_to_string_exceptionless(test_files[i]); + if (!ok) { + return false; + } + } + return true; +} + +bool empty_object() { + TEST_START(); + singlestage::parser parser; + auto arr_json = R"({})"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(arr_json).get(doc)); + std::string_view serial; + ASSERT_SUCCESS( simdjson::to_json_string(doc).get(serial)); + ASSERT_EQUAL(serial, R"({})"); + TEST_SUCCEED(); +} + +bool empty_array() { + TEST_START(); + singlestage::parser parser; + auto arr_json = R"([])"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(arr_json).get(doc)); + std::string_view serial; + ASSERT_SUCCESS( simdjson::to_json_string(doc).get(serial)); + ASSERT_EQUAL(serial, R"([])"); + TEST_SUCCEED(); +} + +bool single_digit_document() { + TEST_START(); + singlestage::parser parser; + auto arr_json = R"(9)"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(arr_json).get(doc) ); + std::string_view serial; + ASSERT_SUCCESS( simdjson::to_json_string(doc).get(serial)); + ASSERT_EQUAL(serial, R"(9)"); + TEST_SUCCEED(); +} + +bool single_string_document() { + TEST_START(); + singlestage::parser parser; + auto arr_json = R"("")"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(arr_json).get(doc) ); + std::string_view serial; + ASSERT_SUCCESS( simdjson::to_json_string(doc).get(serial)); + ASSERT_EQUAL(serial, R"("")"); + TEST_SUCCEED(); +} + +bool at_start_array() { + TEST_START(); + singlestage::parser parser; + auto arr_json = R"( [111,2,3,5] )"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(arr_json).get(doc) ); + singlestage::array array; + ASSERT_SUCCESS( doc.get_array().get(array) ); + std::string_view serial; + ASSERT_SUCCESS( simdjson::to_json_string(array).get(serial)); + ASSERT_EQUAL(serial, "[111,2,3,5]"); + TEST_SUCCEED(); +} + + +bool at_start_object() { + TEST_START(); + singlestage::parser parser; + auto arr_json = R"( {"a":1, "b":2, "c": 3 } )"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(arr_json).get(doc) ); + singlestage::object object; + ASSERT_SUCCESS( doc.get_object().get(object) ); + std::string_view serial; + ASSERT_SUCCESS( simdjson::to_json_string(object).get(serial)); + ASSERT_EQUAL(serial, R"({"a":1, "b":2, "c": 3 })"); + ASSERT_SUCCESS( simdjson::to_json_string(object).get(serial)); + ASSERT_EQUAL(serial, R"({"a":1, "b":2, "c": 3 })"); + TEST_SUCCEED(); +} + +bool in_middle_array() { + TEST_START(); + singlestage::parser parser; + auto arr_json = R"( [111,{"a":1},3,5] )"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(arr_json).get(doc) ); + singlestage::array array; + ASSERT_SUCCESS( doc.get_array().get(array) ); + auto i = array.begin(); + int64_t x; + ASSERT_SUCCESS( (*i).get_int64().get(x) ); + ASSERT_EQUAL(x, 111); + std::string_view serial; + ASSERT_SUCCESS( simdjson::to_json_string(array).get(serial)); + ASSERT_EQUAL(serial, "[111,{\"a\":1},3,5]"); + ASSERT_SUCCESS( simdjson::to_json_string(doc).get(serial)); + ASSERT_EQUAL(serial, "[111,{\"a\":1},3,5]"); + TEST_SUCCEED(); +} + +bool at_middle_object() { + TEST_START(); + singlestage::parser parser; + auto arr_json = R"( {"a":1, "b":2, "c": 3 } )"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(arr_json).get(doc) ); + singlestage::object object; + ASSERT_SUCCESS( doc.get_object().get(object) ); + int64_t x; + ASSERT_SUCCESS(object["b"].get_int64().get(x)); + ASSERT_EQUAL(x,2); + std::string_view serial; + ASSERT_SUCCESS( simdjson::to_json_string(object).get(serial)); + ASSERT_EQUAL(serial, R"({"a":1, "b":2, "c": 3 })"); + ASSERT_SUCCESS( simdjson::to_json_string(doc).get(serial)); + ASSERT_EQUAL(serial, R"({"a":1, "b":2, "c": 3 })"); + TEST_SUCCEED(); +} + +bool at_middle_object_just_key() { + TEST_START(); + singlestage::parser parser; + auto arr_json = R"( {"a":1, "b":2, "c": 3 } )"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(arr_json).get(doc) ); + singlestage::object object; + ASSERT_SUCCESS( doc.get_object().get(object) ); + singlestage::value x; + ASSERT_SUCCESS(object["b"].get(x)); + std::string_view serial; + ASSERT_SUCCESS( simdjson::to_json_string(object).get(serial)); + ASSERT_EQUAL(serial, R"({"a":1, "b":2, "c": 3 })"); + ASSERT_SUCCESS( simdjson::to_json_string(doc).get(serial)); + ASSERT_EQUAL(serial, R"({"a":1, "b":2, "c": 3 })"); + TEST_SUCCEED(); +} + + +bool at_end_object() { + TEST_START(); + singlestage::parser parser; + auto arr_json = R"( {"a":1, "b":2, "c": 3 } )"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(arr_json).get(doc) ); + singlestage::object object; + ASSERT_SUCCESS( doc.get_object().get(object) ); + int64_t x; + ASSERT_ERROR(object["bcc"].get_int64().get(x), NO_SUCH_FIELD); + std::string_view serial; + ASSERT_SUCCESS( simdjson::to_json_string(object).get(serial)); + ASSERT_EQUAL(serial, R"({"a":1, "b":2, "c": 3 })"); + ASSERT_SUCCESS( simdjson::to_json_string(doc).get(serial)); + ASSERT_EQUAL(serial, R"({"a":1, "b":2, "c": 3 })"); + TEST_SUCCEED(); +} + +bool at_array_end() { + TEST_START(); + singlestage::parser parser; + std::string_view serial; + auto arr_json = R"( [111,2,3,5] )"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(arr_json).get(doc) ); + singlestage::array array; + ASSERT_SUCCESS( doc.get_array().get(array) ); + auto i = array.begin(); + int64_t x; + ASSERT_SUCCESS( (*i).get_int64().get(x) ); + ASSERT_EQUAL(x, 111); + ++i; + ASSERT_SUCCESS( (*i).get_int64().get(x) ); + ASSERT_EQUAL(x, 2); + ++i; + ASSERT_SUCCESS( (*i).get_int64().get(x) ); + ASSERT_EQUAL(x, 3); + ++i; + ASSERT_SUCCESS( (*i).get_int64().get(x) ); + ASSERT_EQUAL(x, 5); + ASSERT_SUCCESS( simdjson::to_json_string(array).get(serial)); + ASSERT_EQUAL(serial, "[111,2,3,5]"); + ASSERT_SUCCESS( simdjson::to_json_string(doc).get(serial)); + ASSERT_EQUAL(serial, "[111,2,3,5]"); + TEST_SUCCEED(); +} + +bool complex_case() { + TEST_START(); + singlestage::parser parser; + auto arr_json = R"( {"array":[1,2,3], "objects":[{"id":1}, {"id":2}, {"id":3}]} )"_padded; + singlestage::document doc; + ASSERT_SUCCESS( parser.iterate(arr_json).get(doc) ); + singlestage::object obj; + ASSERT_SUCCESS( doc.get_object().get(obj) ); + singlestage::array array; + ASSERT_SUCCESS( obj["objects"].get_array().get(array) ); + std::string_view serial; + for(auto v : array) { + singlestage::object object; + ASSERT_SUCCESS( v.get_object().get(object) ); + int64_t x; + ASSERT_SUCCESS(object["id"].get_int64().get(x)); + if(x / 2 * 2 != x) { + ASSERT_SUCCESS( simdjson::to_json_string(object).get(serial)); + ASSERT_EQUAL(serial, "{\"id\":"+std::to_string(x)+"}"); + } + } + ASSERT_SUCCESS( simdjson::to_json_string(array).get(serial)); + ASSERT_EQUAL(serial, R"([{"id":1}, {"id":2}, {"id":3}])"); + ASSERT_SUCCESS( simdjson::to_json_string(obj).get(serial)); + ASSERT_EQUAL(serial, R"({"array":[1,2,3], "objects":[{"id":1}, {"id":2}, {"id":3}]})"); + ASSERT_SUCCESS( simdjson::to_json_string(doc).get(serial)); + ASSERT_EQUAL(serial, R"({"array":[1,2,3], "objects":[{"id":1}, {"id":2}, {"id":3}]})"); + TEST_SUCCEED(); +} + +bool run() { + return + empty_object() && + empty_array() && + single_digit_document() && + single_string_document() && + complex_case() && + at_start_object() && + at_middle_object() && + at_middle_object_just_key() && + at_end_object() && + at_start_array() && + in_middle_array() && + at_array_end() && +#if SIMDJSON_EXCEPTIONS + issue1607() && + minify_demo() && + minify_demo2() && + minify_test() && + car_example() && +#endif // SIMDJSON_EXCEPTIONS + minify_exceptionless_test() && + true; +} + +} // namespace tostring_tests + + + + +int main(int argc, char *argv[]) { + return test_main(argc, argv, tostring_tests::run); +} diff --git a/tests/singlestage/singlestage_twitter_tests.cpp b/tests/singlestage/singlestage_twitter_tests.cpp new file mode 100644 index 000000000..b38474409 --- /dev/null +++ b/tests/singlestage/singlestage_twitter_tests.cpp @@ -0,0 +1,197 @@ +#include +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace twitter_tests { + using namespace std; + + bool twitter_count() { + TEST_START(); + padded_string json; + ASSERT_SUCCESS( padded_string::load(TWITTER_JSON).get(json) ); + ASSERT_TRUE(test_singlestage_doc(json, [&](auto doc_result) { + uint64_t count; + ASSERT_SUCCESS( doc_result["search_metadata"]["count"].get(count) ); + ASSERT_EQUAL( count, 100 ); + return true; + })); + TEST_SUCCEED(); + } +#if SIMDJSON_EXCEPTIONS + bool twitter_example() { + TEST_START(); + padded_string json; + ASSERT_SUCCESS( padded_string::load(TWITTER_JSON).get(json) ); + singlestage::parser parser; + auto doc = parser.iterate(json); + for (singlestage::object tweet : doc["statuses"]) { + uint64_t id = tweet["id"]; + std::string_view text = tweet["text"]; + std::string_view screen_name = tweet["user"]["screen_name"]; + uint64_t retweets = tweet["retweet_count"]; + uint64_t favorites = tweet["favorite_count"]; + (void) id; + (void) text; + (void) retweets; + (void) favorites; + (void) screen_name; + } + TEST_SUCCEED(); + } +#endif // SIMDJSON_EXCEPTIONS + + bool twitter_default_profile() { + TEST_START(); + padded_string json; + ASSERT_SUCCESS( padded_string::load(TWITTER_JSON).get(json) ); + ASSERT_TRUE(test_singlestage_doc(json, [&](auto doc_result) { + // Print users with a default profile. + set default_users; + for (auto tweet : doc_result["statuses"]) { + auto user = tweet["user"].get_object(); + + // We have to get the screen name before default_profile because it appears first + std::string_view screen_name; + ASSERT_SUCCESS( user["screen_name"].get(screen_name) ); + + bool default_profile{}; + ASSERT_SUCCESS( user["default_profile"].get(default_profile) ); + if (default_profile) { + default_users.insert(screen_name); + } + } + ASSERT_EQUAL( default_users.size(), 86 ); + return true; + })); + TEST_SUCCEED(); + } + + bool twitter_image_sizes() { + TEST_START(); + padded_string json; + ASSERT_SUCCESS( padded_string::load(TWITTER_JSON).get(json) ); + ASSERT_TRUE(test_singlestage_doc(json, [&](auto doc_result) { + // Print image names and sizes + set> image_sizes; + for (auto tweet : doc_result["statuses"]) { + auto media = tweet["entities"]["media"]; + if (!media.error()) { + for (auto image : media) { + uint64_t id_val; + std::string_view id_string; + ASSERT_SUCCESS( image["id"].get(id_val) ); + ASSERT_SUCCESS( image["id_str"].get(id_string) ); + std::cout << "id = " << id_val << std::endl; + std::cout << "id_string = " << id_string << std::endl; + + for (auto size : image["sizes"].get_object()) { + std::string_view size_key; + ASSERT_SUCCESS( size.unescaped_key().get(size_key) ); + std::cout << "Type of image size = " << size_key << std::endl; + + uint64_t width, height; + ASSERT_SUCCESS( size.value()["w"].get(width) ); + ASSERT_SUCCESS( size.value()["h"].get(height) ); + image_sizes.insert(make_pair(width, height)); + } + } + } + } + ASSERT_EQUAL( image_sizes.size(), 15 ); + return true; + })); + TEST_SUCCEED(); + } + +#if SIMDJSON_EXCEPTIONS + + bool twitter_count_exception() { + TEST_START(); + padded_string json; + ASSERT_SUCCESS( padded_string::load(TWITTER_JSON).get(json) ); + ASSERT_TRUE(test_singlestage_doc(json, [&](auto doc_result) { + uint64_t count = doc_result["search_metadata"]["count"]; + ASSERT_EQUAL( count, 100 ); + return true; + })); + TEST_SUCCEED(); + } + + bool twitter_default_profile_exception() { + TEST_START(); + padded_string json = padded_string::load(TWITTER_JSON); + ASSERT_TRUE(test_singlestage_doc(json, [&](auto doc_result) { + // Print users with a default profile. + set default_users; + for (auto tweet : doc_result["statuses"]) { + singlestage::object user = tweet["user"]; + + // We have to get the screen name before default_profile because it appears first + std::string_view screen_name = user["screen_name"]; + if (user["default_profile"]) { + default_users.insert(screen_name); + } + } + ASSERT_EQUAL( default_users.size(), 86 ); + return true; + })); + TEST_SUCCEED(); + } + + /* + * Fun fact: id and id_str can differ: + * 505866668485386240 and 505866668485386241. + * Presumably, it is because doubles are used + * at some point in the process and the number + * 505866668485386241 cannot be represented as a double. + * (not our fault) + */ + bool twitter_image_sizes_exception() { + TEST_START(); + padded_string json = padded_string::load(TWITTER_JSON); + ASSERT_TRUE(test_singlestage_doc(json, [&](auto doc_result) { + // Print image names and sizes + set> image_sizes; + for (auto tweet : doc_result["statuses"]) { + auto media = tweet["entities"]["media"]; + if (!media.error()) { + for (auto image : media) { + std::cout << "id = " << uint64_t(image["id"]) << std::endl; + std::cout << "id_string = " << std::string_view(image["id_str"]) << std::endl; + for (auto size : image["sizes"].get_object()) { + std::cout << "Type of image size = " << std::string_view(size.unescaped_key()) << std::endl; + // NOTE: the uint64_t is required so that each value is actually parsed before the pair is created + image_sizes.insert(make_pair(size.value()["w"], size.value()["h"])); + } + } + } + } + ASSERT_EQUAL( image_sizes.size(), 15 ); + return true; + })); + TEST_SUCCEED(); + } + +#endif // SIMDJSON_EXCEPTIONS + + bool run() { + return + twitter_count() && + twitter_default_profile() && + twitter_image_sizes() && +#if SIMDJSON_EXCEPTIONS + twitter_count_exception() && + twitter_example() && + twitter_default_profile_exception() && + twitter_image_sizes_exception() && +#endif // SIMDJSON_EXCEPTIONS + true; + } + +} // namespace twitter_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, twitter_tests::run); +} diff --git a/tests/singlestage/singlestage_wrong_type_error_tests.cpp b/tests/singlestage/singlestage_wrong_type_error_tests.cpp new file mode 100644 index 000000000..8b1757ef6 --- /dev/null +++ b/tests/singlestage/singlestage_wrong_type_error_tests.cpp @@ -0,0 +1,176 @@ +#include "simdjson.h" +#include "test_singlestage.h" + +using namespace simdjson; + +namespace wrong_type_error_tests { + using namespace std; + +#define TEST_CAST_ERROR(JSON, TYPE, ERROR) \ + std::cout << "- Subtest: get_" << (#TYPE) << "() - JSON: " << (JSON) << std::endl; \ + if (!test_singlestage_doc((JSON##_padded), [&](auto doc_result) { \ + ASSERT_ERROR( doc_result.get_##TYPE(), (ERROR) ); \ + return true; \ + })) { \ + return false; \ + } \ + { \ + padded_string a_json(std::string(R"({ "a": )") + JSON + " }"); \ + std::cout << R"(- Subtest: get_)" << (#TYPE) << "() - JSON: " << a_json << std::endl; \ + if (!test_singlestage_doc(a_json, [&](auto doc_result) { \ + ASSERT_ERROR( doc_result["a"].get_##TYPE(), (ERROR) ); \ + return true; \ + })) { \ + return false; \ + }; \ + } + + bool wrong_type_array() { + TEST_START(); + TEST_CAST_ERROR("[]", object, INCORRECT_TYPE); + TEST_CAST_ERROR("[]", bool, INCORRECT_TYPE); + TEST_CAST_ERROR("[]", int64, INCORRECT_TYPE); + TEST_CAST_ERROR("[]", uint64, INCORRECT_TYPE); + TEST_CAST_ERROR("[]", double, INCORRECT_TYPE); + TEST_CAST_ERROR("[]", string, INCORRECT_TYPE); + TEST_CAST_ERROR("[]", raw_json_string, INCORRECT_TYPE); + TEST_SUCCEED(); + } + + bool wrong_type_object() { + TEST_START(); + TEST_CAST_ERROR("{}", array, INCORRECT_TYPE); + TEST_CAST_ERROR("{}", bool, INCORRECT_TYPE); + TEST_CAST_ERROR("{}", int64, INCORRECT_TYPE); + TEST_CAST_ERROR("{}", uint64, INCORRECT_TYPE); + TEST_CAST_ERROR("{}", double, INCORRECT_TYPE); + TEST_CAST_ERROR("{}", string, INCORRECT_TYPE); + TEST_CAST_ERROR("{}", raw_json_string, INCORRECT_TYPE); + TEST_SUCCEED(); + } + + bool wrong_type_true() { + TEST_START(); + TEST_CAST_ERROR("true", array, INCORRECT_TYPE); + TEST_CAST_ERROR("true", object, INCORRECT_TYPE); + TEST_CAST_ERROR("true", int64, INCORRECT_TYPE); + TEST_CAST_ERROR("true", uint64, INCORRECT_TYPE); + TEST_CAST_ERROR("true", double, INCORRECT_TYPE); + TEST_CAST_ERROR("true", string, INCORRECT_TYPE); + TEST_CAST_ERROR("true", raw_json_string, INCORRECT_TYPE); + TEST_SUCCEED(); + } + + bool wrong_type_false() { + TEST_START(); + TEST_CAST_ERROR("false", array, INCORRECT_TYPE); + TEST_CAST_ERROR("false", object, INCORRECT_TYPE); + TEST_CAST_ERROR("false", int64, INCORRECT_TYPE); + TEST_CAST_ERROR("false", uint64, INCORRECT_TYPE); + TEST_CAST_ERROR("false", double, INCORRECT_TYPE); + TEST_CAST_ERROR("false", string, INCORRECT_TYPE); + TEST_CAST_ERROR("false", raw_json_string, INCORRECT_TYPE); + TEST_SUCCEED(); + } + + bool wrong_type_null() { + TEST_START(); + TEST_CAST_ERROR("null", array, INCORRECT_TYPE); + TEST_CAST_ERROR("null", object, INCORRECT_TYPE); + TEST_CAST_ERROR("null", int64, INCORRECT_TYPE); + TEST_CAST_ERROR("null", uint64, INCORRECT_TYPE); + TEST_CAST_ERROR("null", double, INCORRECT_TYPE); + TEST_CAST_ERROR("null", string, INCORRECT_TYPE); + TEST_CAST_ERROR("null", raw_json_string, INCORRECT_TYPE); + TEST_SUCCEED(); + } + + bool wrong_type_1() { + TEST_START(); + TEST_CAST_ERROR("1", array, INCORRECT_TYPE); + TEST_CAST_ERROR("1", object, INCORRECT_TYPE); + TEST_CAST_ERROR("1", bool, INCORRECT_TYPE); + TEST_CAST_ERROR("1", string, INCORRECT_TYPE); + TEST_CAST_ERROR("1", raw_json_string, INCORRECT_TYPE); + TEST_SUCCEED(); + } + + bool wrong_type_negative_1() { + TEST_START(); + TEST_CAST_ERROR("-1", array, INCORRECT_TYPE); + TEST_CAST_ERROR("-1", object, INCORRECT_TYPE); + TEST_CAST_ERROR("-1", bool, INCORRECT_TYPE); + TEST_CAST_ERROR("-1", uint64, INCORRECT_TYPE); + TEST_CAST_ERROR("-1", string, INCORRECT_TYPE); + TEST_CAST_ERROR("-1", raw_json_string, INCORRECT_TYPE); + TEST_SUCCEED(); + } + + bool wrong_type_float() { + TEST_START(); + TEST_CAST_ERROR("1.1", array, INCORRECT_TYPE); + TEST_CAST_ERROR("1.1", object, INCORRECT_TYPE); + TEST_CAST_ERROR("1.1", bool, INCORRECT_TYPE); + TEST_CAST_ERROR("1.1", int64, INCORRECT_TYPE); + TEST_CAST_ERROR("1.1", uint64, INCORRECT_TYPE); + TEST_CAST_ERROR("1.1", string, INCORRECT_TYPE); + TEST_CAST_ERROR("1.1", raw_json_string, INCORRECT_TYPE); + TEST_SUCCEED(); + } + + bool wrong_type_negative_int64_overflow() { + TEST_START(); + TEST_CAST_ERROR("-9223372036854775809", array, INCORRECT_TYPE); + TEST_CAST_ERROR("-9223372036854775809", object, INCORRECT_TYPE); + TEST_CAST_ERROR("-9223372036854775809", bool, INCORRECT_TYPE); + TEST_CAST_ERROR("-9223372036854775809", int64, INCORRECT_TYPE); + TEST_CAST_ERROR("-9223372036854775809", uint64, INCORRECT_TYPE); + TEST_CAST_ERROR("-9223372036854775809", string, INCORRECT_TYPE); + TEST_CAST_ERROR("-9223372036854775809", raw_json_string, INCORRECT_TYPE); + TEST_SUCCEED(); + } + + bool wrong_type_int64_overflow() { + TEST_START(); + TEST_CAST_ERROR("9223372036854775808", array, INCORRECT_TYPE); + TEST_CAST_ERROR("9223372036854775808", object, INCORRECT_TYPE); + TEST_CAST_ERROR("9223372036854775808", bool, INCORRECT_TYPE); + TEST_CAST_ERROR("9223372036854775808", int64, INCORRECT_TYPE); + TEST_CAST_ERROR("9223372036854775808", string, INCORRECT_TYPE); + TEST_CAST_ERROR("9223372036854775808", raw_json_string, INCORRECT_TYPE); + TEST_SUCCEED(); + } + + bool wrong_type_uint64_overflow() { + TEST_START(); + TEST_CAST_ERROR("18446744073709551616", array, INCORRECT_TYPE); + TEST_CAST_ERROR("18446744073709551616", object, INCORRECT_TYPE); + TEST_CAST_ERROR("18446744073709551616", bool, INCORRECT_TYPE); + TEST_CAST_ERROR("18446744073709551616", int64, INCORRECT_TYPE); + TEST_CAST_ERROR("18446744073709551616", uint64, INCORRECT_TYPE); + TEST_CAST_ERROR("18446744073709551616", string, INCORRECT_TYPE); + TEST_CAST_ERROR("18446744073709551616", raw_json_string, INCORRECT_TYPE); + TEST_SUCCEED(); + } + + bool run() { + return + wrong_type_1() && + wrong_type_array() && + wrong_type_false() && + wrong_type_float() && + wrong_type_int64_overflow() && + wrong_type_negative_1() && + wrong_type_negative_int64_overflow() && + wrong_type_null() && + wrong_type_object() && + wrong_type_true() && + wrong_type_uint64_overflow() && + true; + } + +} // namespace wrong_type_error_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, wrong_type_error_tests::run); +} diff --git a/tests/singlestage/test_singlestage.h b/tests/singlestage/test_singlestage.h new file mode 100644 index 000000000..47929005c --- /dev/null +++ b/tests/singlestage/test_singlestage.h @@ -0,0 +1,87 @@ +#ifndef SINGLESTAGE_TEST_SINGLESTAGE_H +#define SINGLESTAGE_TEST_SINGLESTAGE_H + +#include +#include +#include "simdjson.h" +#include "cast_tester.h" +#include "test_macros.h" + +template +bool test_singlestage(simdjson::singlestage::parser &parser, const simdjson::padded_string &json, const F& f) { + auto doc = parser.iterate(json); + T val{}; + ASSERT_SUCCESS( doc.get(val) ); + return f(val); +} +template +bool test_singlestage(const simdjson::padded_string &json, const F& f) { + simdjson::singlestage::parser parser; + return test_singlestage(parser, json, f); +} + +template +bool test_singlestage_doc(simdjson::singlestage::parser &parser, const simdjson::padded_string &json, const F& f) { + return f(parser.iterate(json)); +} +template +bool test_singlestage_doc(const simdjson::padded_string &json, const F& f) { + simdjson::singlestage::parser parser; + return test_singlestage_doc(parser, json, f); +} + +#define SINGLESTAGE_SUBTEST(NAME, JSON, TEST) \ +{ \ + std::cout << "- Subtest " << NAME << " - JSON: " << (JSON) << " ..." << std::endl; \ + if (!test_singlestage_doc(JSON##_padded, [&](auto doc) { \ + return (TEST); \ + })) { \ + return false; \ + } \ +} + +const size_t AMAZON_CELLPHONES_NDJSON_DOC_COUNT = 793; +#define SIMDJSON_SHOW_DEFINE(x) printf("%s=%s\n", #x, SIMDJSON_STRINGIFY(x)) + +template +int test_main(int argc, char *argv[], const F& test_function) { + std::cout << std::unitbuf; + int c; + while ((c = getopt(argc, argv, "a:")) != -1) { + switch (c) { + case 'a': { + const simdjson::implementation *impl = simdjson::get_available_implementations()[optarg]; + if (!impl) { + std::fprintf(stderr, "Unsupported architecture value -a %s\n", optarg); + return EXIT_FAILURE; + } + simdjson::get_active_implementation() = impl; + break; + } + default: + std::fprintf(stderr, "Unexpected argument %c\n", c); + return EXIT_FAILURE; + } + } + + // this is put here deliberately to check that the documentation is correct (README), + // should this fail to compile, you should update the documentation: + if (simdjson::get_active_implementation()->name() == "unsupported") { + std::printf("unsupported CPU\n"); + std::abort(); + } + // We want to know what we are testing. + std::cout << "builtin_implementation -- " << simdjson::builtin_implementation()->name() << std::endl; + std::cout << "------------------------------------------------------------" << std::endl; + + std::cout << "Running tests." << std::endl; + if (test_function()) { + std::cout << "Success!" << std::endl; + return EXIT_SUCCESS; + } else { + std::cerr << "FAILED." << std::endl; + return EXIT_FAILURE; + } +} + +#endif // SINGLESTAGE_TEST_SINGLESTAGE_H diff --git a/tests/test_macros.h b/tests/test_macros.h index 7df60dc0b..a5e8c0752 100644 --- a/tests/test_macros.h +++ b/tests/test_macros.h @@ -37,6 +37,10 @@ template<> simdjson_inline bool equals_expected(simdjson::ondemand::raw_json_string actual, const char * expected) { return actual == expected; } +template<> +simdjson_inline bool equals_expected(simdjson::singlestage::raw_json_string actual, const char * expected) { + return actual == expected; +} simdjson_inline simdjson::error_code to_error_code(simdjson::error_code error) { return error;