diff --git a/.appveyor.yml b/.appveyor.yml index ec37094d3..701e9b865 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -13,24 +13,24 @@ environment: matrix: - job_name: VS2019 - CMAKE_ARGS: -A %Platform% + CMAKE_ARGS: -A %Platform% - job_name: VS2019ARM CMAKE_ARGS: -A ARM64 -DCMAKE_CROSSCOMPILING=1 -D SIMDJSON_GOOGLE_BENCHMARKS=OFF # Does Google Benchmark builds under VS ARM? - job_name: VS2017 (Static, No Threads) image: Visual Studio 2017 - CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_ENABLE_THREADS=OFF + CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_ENABLE_THREADS=OFF CTEST_ARGS: -LE explicitonly - job_name: VS2019 (Win32) platform: Win32 - CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway. - CTEST_ARGS: -E "checkperf|ondemand_basictests" + CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway. + CTEST_ARGS: -LE explicitonly - job_name: VS2019 (Win32, No Exceptions) platform: Win32 - CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF - CTEST_ARGS: -E "checkperf|ondemand_basictests" + CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF + CTEST_ARGS: -LE explicitonly - job_name: VS2015 image: Visual Studio 2015 - CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_ENABLE_THREADS=OFF + CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_ENABLE_THREADS=OFF CTEST_ARGS: -LE explicitonly build_script: diff --git a/.github/workflows/mingw-ci.yml b/.github/workflows/mingw-ci.yml index 55c32e55a..957edbc59 100644 --- a/.github/workflows/mingw-ci.yml +++ b/.github/workflows/mingw-ci.yml @@ -61,5 +61,5 @@ jobs: mkdir build32 cd build32 cmake -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. - cmake --build . --target parse_many_test jsoncheck basictests ondemand_basictests numberparsingcheck stringparsingcheck errortests integer_tests pointercheck --verbose - ctest -R "(parse_many_test|jsoncheck|basictests|stringparsingcheck|numberparsingcheck|errortests|integer_tests|pointercheck)" --output-on-failure + cmake --build . --target acceptance_tests --verbose + ctest -L acceptance --output-on-failure diff --git a/.github/workflows/mingw64-ci.yml b/.github/workflows/mingw64-ci.yml index cf2fcab6c..0dd6d5d4a 100644 --- a/.github/workflows/mingw64-ci.yml +++ b/.github/workflows/mingw64-ci.yml @@ -61,11 +61,11 @@ jobs: mkdir build64 cd build64 cmake -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. - cmake --build . --target parse_many_test jsoncheck basictests ondemand_basictests numberparsingcheck stringparsingcheck errortests integer_tests pointercheck --verbose - ctest -R "(parse_many_test|jsoncheck|basictests|stringparsingcheck|numberparsingcheck|errortests|integer_tests|pointercheck)" --output-on-failure + cmake --build . --target acceptance_tests --verbose + ctest -L acceptance --output-on-failure cd .. mkdir build64debug cd build64debug cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. - cmake --build . --target parse_many_test jsoncheck basictests ondemand_basictests numberparsingcheck stringparsingcheck errortests integer_tests pointercheck --verbose - ctest -R "(parse_many_test|jsoncheck|basictests|stringparsingcheck|numberparsingcheck|errortests|integer_tests|pointercheck)" --output-on-failure + cmake --build . --target acceptance_tests --verbose + ctest -L acceptance --output-on-failure diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index f15c73ff8..b3c796b2c 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -1,13 +1,15 @@ include_directories( . linux ) link_libraries(simdjson-windows-headers test-data) - +# bench_sax links against the source if (TARGET benchmark::benchmark) add_executable(bench_sax bench_sax.cpp) target_link_libraries(bench_sax PRIVATE simdjson-internal-flags simdjson-include-source benchmark::benchmark) endif (TARGET benchmark::benchmark) +# Everything else links against simdjson proper link_libraries(simdjson simdjson-flags) + add_executable(benchfeatures benchfeatures.cpp) add_executable(get_corpus_benchmark get_corpus_benchmark.cpp) add_executable(perfdiff perfdiff.cpp) @@ -42,6 +44,7 @@ endif() if (TARGET benchmark::benchmark) link_libraries(benchmark::benchmark) + add_subdirectory(largerandom) add_executable(bench_parse_call bench_parse_call.cpp) add_executable(bench_dom_api bench_dom_api.cpp) add_executable(bench_ondemand bench_ondemand.cpp) diff --git a/benchmark/bench_ondemand.cpp b/benchmark/bench_ondemand.cpp index 98fd42c8d..04757ccc3 100644 --- a/benchmark/bench_ondemand.cpp +++ b/benchmark/bench_ondemand.cpp @@ -12,6 +12,7 @@ SIMDJSON_POP_DISABLE_WARNINGS #include "partial_tweets/dom.h" #include "largerandom/ondemand.h" +#include "largerandom/ondemand_unordered.h" // #include "largerandom/iter.h" #include "largerandom/dom.h" diff --git a/benchmark/distinctuserid/ondemand.h b/benchmark/distinctuserid/ondemand.h index 933e630fe..fd3963401 100644 --- a/benchmark/distinctuserid/ondemand.h +++ b/benchmark/distinctuserid/ondemand.h @@ -33,15 +33,15 @@ simdjson_really_inline bool OnDemand::Run(const padded_string &json) { ids.clear(); // Walk the document, parsing as we go auto doc = parser.iterate(json); - for (ondemand::object tweet : doc["statuses"]) { + for (ondemand::object tweet : doc.find_field("statuses")) { // We believe that all statuses have a matching // user, and we are willing to throw when they do not. - ids.push_back(tweet["user"]["id"]); + ids.push_back(tweet.find_field("user").find_field("id")); // Not all tweets have a "retweeted_status", but when they do // we want to go and find the user within. - auto retweet = tweet["retweeted_status"]; + auto retweet = tweet.find_field("retweeted_status"); if(!retweet.error()) { - ids.push_back(retweet["user"]["id"]); + ids.push_back(retweet.find_field("user").find_field("id")); } } remove_duplicates(ids); diff --git a/benchmark/find_tweet/ondemand.h b/benchmark/find_tweet/ondemand.h index 0f8576c0f..92e72dc28 100644 --- a/benchmark/find_tweet/ondemand.h +++ b/benchmark/find_tweet/ondemand.h @@ -33,9 +33,9 @@ simdjson_really_inline bool OnDemand::Run(const padded_string &json) { text = ""; // Walk the document, parsing as we go auto doc = parser.iterate(json); - for (ondemand::object tweet : doc["statuses"]) { - if (uint64_t(tweet["id"]) == TWEET_ID) { - text = tweet["text"]; + for (ondemand::object tweet : doc.find_field("statuses")) { + if (uint64_t(tweet.find_field("id")) == TWEET_ID) { + text = tweet.find_field("text"); return true; } } diff --git a/benchmark/kostya/ondemand.h b/benchmark/kostya/ondemand.h index 369ffedb1..1da401205 100644 --- a/benchmark/kostya/ondemand.h +++ b/benchmark/kostya/ondemand.h @@ -27,8 +27,8 @@ simdjson_really_inline bool OnDemand::Run(const padded_string &json) { using std::endl; auto doc = parser.iterate(json); - for (ondemand::object coord : doc["coordinates"]) { - container.emplace_back(my_point{coord["x"], coord["y"], coord["z"]}); + for (ondemand::object coord : doc.find_field("coordinates")) { + container.emplace_back(my_point{coord.find_field("x"), coord.find_field("y"), coord.find_field("z")}); } return true; @@ -56,10 +56,10 @@ simdjson_really_inline bool OnDemand::Run(const padded_string &json) { count = 0; auto doc = parser.iterate(json); - for (ondemand::object coord : doc["coordinates"]) { - sum.x += double(coord["x"]); - sum.y += double(coord["y"]); - sum.z += double(coord["z"]); + for (ondemand::object coord : doc.find_field("coordinates")) { + sum.x += double(coord.find_field("x")); + sum.y += double(coord.find_field("y")); + sum.z += double(coord.find_field("z")); count++; } diff --git a/benchmark/largerandom/CMakeLists.txt b/benchmark/largerandom/CMakeLists.txt new file mode 100644 index 000000000..1ca63b5fd --- /dev/null +++ b/benchmark/largerandom/CMakeLists.txt @@ -0,0 +1,5 @@ +if (TARGET benchmark::benchmark) + link_libraries(benchmark::benchmark) + add_executable(bench_ondemand_largerandom bench_ondemand_largerandom.cpp) + add_executable(bench_ondemand_unordered_largerandom bench_ondemand_unordered_largerandom.cpp) +endif() diff --git a/benchmark/largerandom/bench_ondemand_largerandom.cpp b/benchmark/largerandom/bench_ondemand_largerandom.cpp new file mode 100644 index 000000000..e1f35f6b2 --- /dev/null +++ b/benchmark/largerandom/bench_ondemand_largerandom.cpp @@ -0,0 +1,14 @@ +#include "simdjson.h" +#include +#include +#include +#include +SIMDJSON_PUSH_DISABLE_ALL_WARNINGS +#include +SIMDJSON_POP_DISABLE_WARNINGS + +#define BENCHMARK_NO_DOM + +#include "largerandom/ondemand.h" + +BENCHMARK_MAIN(); diff --git a/benchmark/largerandom/bench_ondemand_unordered_largerandom.cpp b/benchmark/largerandom/bench_ondemand_unordered_largerandom.cpp new file mode 100644 index 000000000..54b9c70fd --- /dev/null +++ b/benchmark/largerandom/bench_ondemand_unordered_largerandom.cpp @@ -0,0 +1,14 @@ +#include "simdjson.h" +#include +#include +#include +#include +SIMDJSON_PUSH_DISABLE_ALL_WARNINGS +#include +SIMDJSON_POP_DISABLE_WARNINGS + +#define BENCHMARK_NO_DOM + +#include "largerandom/ondemand_unordered.h" + +BENCHMARK_MAIN(); diff --git a/benchmark/largerandom/dom.h b/benchmark/largerandom/dom.h index 4148eeee0..a2642c04c 100644 --- a/benchmark/largerandom/dom.h +++ b/benchmark/largerandom/dom.h @@ -32,38 +32,6 @@ simdjson_really_inline bool Dom::Run(const padded_string &json) { BENCHMARK_TEMPLATE(LargeRandom, Dom); -namespace sum { - -class Dom { -public: - simdjson_really_inline bool Run(const padded_string &json); - - simdjson_really_inline my_point &Result() { return sum; } - simdjson_really_inline size_t ItemCount() { return count; } - -private: - dom::parser parser{}; - my_point sum{}; - size_t count{}; -}; - -simdjson_really_inline bool Dom::Run(const padded_string &json) { - sum = { 0, 0, 0 }; - count = 0; - - for (auto coord : parser.parse(json)) { - sum.x += double(coord["x"]); - sum.y += double(coord["y"]); - sum.z += double(coord["z"]); - count++; - } - - return true; -} - -BENCHMARK_TEMPLATE(LargeRandomSum, Dom); - -} // namespace sum } // namespace largerandom #endif // SIMDJSON_EXCEPTIONS \ No newline at end of file diff --git a/benchmark/largerandom/iter.h b/benchmark/largerandom/iter.h index 10ab2354e..bb4b2cb8e 100644 --- a/benchmark/largerandom/iter.h +++ b/benchmark/largerandom/iter.h @@ -48,45 +48,6 @@ simdjson_really_inline bool Iter::Run(const padded_string &json) { BENCHMARK_TEMPLATE(LargeRandom, Iter); - -namespace sum { - -class Iter { -public: - simdjson_really_inline bool Run(const padded_string &json); - - simdjson_really_inline my_point &Result() { return sum; } - simdjson_really_inline size_t ItemCount() { return count; } - -private: - ondemand::parser parser{}; - my_point sum{}; - size_t count{}; -}; - -simdjson_really_inline bool Iter::Run(const padded_string &json) { - sum = {0,0,0}; - count = 0; - - auto iter = parser.iterate_raw(json).value(); - if (!iter.start_array()) { return false; } - do { - if (!iter.start_object() || iter.field_key().value() != "x" || iter.field_value()) { return false; } - sum.x += iter.consume_double(); - if (!iter.has_next_field() || iter.field_key().value() != "y" || iter.field_value()) { return false; } - sum.y += iter.consume_double(); - if (!iter.has_next_field() || iter.field_key().value() != "z" || iter.field_value()) { return false; } - sum.z += iter.consume_double(); - if (*iter.advance() != '}') { return false; } - count++; - } while (iter.has_next_element()); - - return true; -} - -BENCHMARK_TEMPLATE(LargeRandomSum, Iter); - -} // namespace sum } // namespace largerandom #endif // SIMDJSON_EXCEPTIONS diff --git a/benchmark/largerandom/largerandom.h b/benchmark/largerandom/largerandom.h index c8bf3d7a8..e7fc6d5ad 100644 --- a/benchmark/largerandom/largerandom.h +++ b/benchmark/largerandom/largerandom.h @@ -8,9 +8,6 @@ namespace largerandom { template static void LargeRandom(benchmark::State &state); -namespace sum { -template static void LargeRandomSum(benchmark::State &state); -} using namespace simdjson; @@ -59,22 +56,21 @@ simdjson_unused static std::ostream &operator<<(std::ostream &o, const my_point // #include #include "event_counter.h" +#ifndef BENCHMARK_NO_DOM #include "dom.h" +#endif #include "json_benchmark.h" namespace largerandom { template static void LargeRandom(benchmark::State &state) { +#ifdef BENCHMARK_NO_DOM + JsonBenchmark(state, get_built_json_array()); +#else JsonBenchmark(state, get_built_json_array()); +#endif } -namespace sum { - -template static void LargeRandomSum(benchmark::State &state) { - JsonBenchmark(state, get_built_json_array()); -} - -} } // namespace largerandom #endif // SIMDJSON_EXCEPTIONS diff --git a/benchmark/largerandom/ondemand.h b/benchmark/largerandom/ondemand.h index 42c4a9b99..f3b5fcaac 100644 --- a/benchmark/largerandom/ondemand.h +++ b/benchmark/largerandom/ondemand.h @@ -25,7 +25,7 @@ simdjson_really_inline bool OnDemand::Run(const padded_string &json) { auto doc = parser.iterate(json); for (ondemand::object coord : doc) { - container.emplace_back(my_point{coord["x"], coord["y"], coord["z"]}); + container.emplace_back(my_point{coord.find_field("x"), coord.find_field("y"), coord.find_field("z")}); } return true; @@ -33,39 +33,6 @@ simdjson_really_inline bool OnDemand::Run(const padded_string &json) { BENCHMARK_TEMPLATE(LargeRandom, OnDemand); - -namespace sum { - -class OnDemand { -public: - simdjson_really_inline bool Run(const padded_string &json); - simdjson_really_inline my_point &Result() { return sum; } - simdjson_really_inline size_t ItemCount() { return count; } - -private: - ondemand::parser parser{}; - my_point sum{}; - size_t count{}; -}; - -simdjson_really_inline bool OnDemand::Run(const padded_string &json) { - sum = {0,0,0}; - count = 0; - - auto doc = parser.iterate(json); - for (ondemand::object coord : doc.get_array()) { - sum.x += double(coord["x"]); - sum.y += double(coord["y"]); - sum.z += double(coord["z"]); - count++; - } - - return true; -} - -BENCHMARK_TEMPLATE(LargeRandomSum, OnDemand); - -} // namespace sum } // namespace largerandom #endif // SIMDJSON_EXCEPTIONS diff --git a/benchmark/largerandom/ondemand_unordered.h b/benchmark/largerandom/ondemand_unordered.h new file mode 100644 index 000000000..553af040f --- /dev/null +++ b/benchmark/largerandom/ondemand_unordered.h @@ -0,0 +1,38 @@ +#pragma once + +#if SIMDJSON_EXCEPTIONS + +#include "largerandom.h" + +namespace largerandom { + +using namespace simdjson; +using namespace simdjson::builtin; + +class OnDemandUnordered { +public: + simdjson_really_inline bool Run(const padded_string &json); + simdjson_really_inline const std::vector &Result() { return container; } + simdjson_really_inline size_t ItemCount() { return container.size(); } + +private: + ondemand::parser parser{}; + std::vector container{}; +}; + +simdjson_really_inline bool OnDemandUnordered::Run(const padded_string &json) { + container.clear(); + + auto doc = parser.iterate(json); + for (ondemand::object coord : doc) { + container.emplace_back(my_point{coord["x"], coord["y"], coord["z"]}); + } + + return true; +} + +BENCHMARK_TEMPLATE(LargeRandom, OnDemandUnordered); + +} // namespace largerandom + +#endif // SIMDJSON_EXCEPTIONS diff --git a/benchmark/partial_tweets/ondemand.h b/benchmark/partial_tweets/ondemand.h index 8ba2079bb..def6e8323 100644 --- a/benchmark/partial_tweets/ondemand.h +++ b/benchmark/partial_tweets/ondemand.h @@ -32,7 +32,7 @@ private: } simdjson_really_inline twitter_user read_user(ondemand::object user) { - return { user["id"], user["screen_name"] }; + return { user.find_field("id"), user.find_field("screen_name") }; } static inline bool displayed_implementation = false; @@ -43,15 +43,15 @@ simdjson_really_inline bool OnDemand::Run(const padded_string &json) { // Walk the document, parsing the tweets as we go auto doc = parser.iterate(json); - for (ondemand::object tweet : doc["statuses"]) { + for (ondemand::object tweet : doc.find_field("statuses")) { tweets.emplace_back(partial_tweets::tweet{ - tweet["created_at"], - tweet["id"], - tweet["text"], - nullable_int(tweet["in_reply_to_status_id"]), - read_user(tweet["user"]), - tweet["retweet_count"], - tweet["favorite_count"] + tweet.find_field("created_at"), + tweet.find_field("id"), + tweet.find_field("text"), + nullable_int(tweet.find_field("in_reply_to_status_id")), + read_user(tweet.find_field("user")), + tweet.find_field("retweet_count"), + tweet.find_field("favorite_count") }); } return true; diff --git a/cmake/add_compile_only_test.cmake b/cmake/add_compile_only_test.cmake new file mode 100644 index 000000000..40ccf7e47 --- /dev/null +++ b/cmake/add_compile_only_test.cmake @@ -0,0 +1,8 @@ +function(add_compile_only_test TEST_NAME) + add_test( + NAME ${TEST_NAME} + COMMAND ${CMAKE_COMMAND} --build . --target ${TEST_NAME} --config $ + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + ) + set_target_properties(${TEST_NAME} PROPERTIES EXCLUDE_FROM_ALL TRUE EXCLUDE_FROM_DEFAULT_BUILD TRUE) +endfunction() \ No newline at end of file diff --git a/cmake/add_cpp_test.cmake b/cmake/add_cpp_test.cmake index b41087fa5..c88a6940e 100644 --- a/cmake/add_cpp_test.cmake +++ b/cmake/add_cpp_test.cmake @@ -3,12 +3,12 @@ # SOURCES defaults to testname.cpp if not specified. function(add_cpp_test TEST_NAME) # Parse arguments - cmake_parse_arguments(PARSE_ARGV 1 ARGS "COMPILE_ONLY;LIBRARY;WILL_FAIL" "" "SOURCES;LABELS") + cmake_parse_arguments(PARSE_ARGV 1 ARGS "COMPILE_ONLY;LIBRARY;WILL_FAIL" "" "SOURCES;LABELS;DEPENDENCY_OF") if (NOT ARGS_SOURCES) list(APPEND ARGS_SOURCES ${TEST_NAME}.cpp) endif() if (ARGS_COMPILE_ONLY) - list(APPEND ${ARGS_LABELS} compile) + list(APPEND ${ARGS_LABELS} compile_only) endif() # Add the compile target @@ -28,22 +28,29 @@ function(add_cpp_test TEST_NAME) set_target_properties(${TEST_NAME} PROPERTIES EXCLUDE_FROM_ALL TRUE EXCLUDE_FROM_DEFAULT_BUILD TRUE) else() add_test(${TEST_NAME} ${TEST_NAME}) + + # Add to