From 90409897eb9deafca46f047dec462564db2722ca Mon Sep 17 00:00:00 2001 From: John Keiser Date: Mon, 22 Mar 2021 11:03:00 -0700 Subject: [PATCH] Add __SIMDJSON_CHECK_EOF feature flag --- .circleci/config.yml | 26 ++++++++++++------- include/simdjson/common_defs.h | 6 +++++ .../generic/ondemand/json_iterator-inl.h | 4 ++- tests/ondemand/ondemand_array_error_tests.cpp | 12 +++++++++ .../ondemand/ondemand_object_error_tests.cpp | 4 +++ 5 files changed, 42 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c571aabd4..c96ee86ee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,7 @@ executors: environment: CXX: g++-8 CC: gcc-8 - BUILD_FLAGS: + CMAKE_BUILD_FLAGS: CTEST_FLAGS: --output-on-failure gcc9: @@ -20,7 +20,7 @@ executors: environment: CXX: g++-9 CC: gcc-9 - BUILD_FLAGS: + CMAKE_BUILD_FLAGS: CTEST_FLAGS: --output-on-failure gcc10: @@ -29,7 +29,7 @@ executors: environment: CXX: g++-10 CC: gcc-10 - BUILD_FLAGS: + CMAKE_BUILD_FLAGS: CTEST_FLAGS: --output-on-failure clang10: @@ -38,7 +38,7 @@ executors: environment: CXX: clang++-10 CC: clang-10 - BUILD_FLAGS: + CMAKE_BUILD_FLAGS: CTEST_FLAGS: --output-on-failure clang9: @@ -47,7 +47,7 @@ executors: environment: CXX: clang++-9 CC: clang-9 - BUILD_FLAGS: + CMAKE_BUILD_FLAGS: CTEST_FLAGS: --output-on-failure clang6: @@ -56,7 +56,7 @@ executors: environment: CXX: clang++-6.0 CC: clang-6.0 - BUILD_FLAGS: + CMAKE_BUILD_FLAGS: CTEST_FLAGS: --output-on-failure # Reusable test commands (and initializer for clang 6) @@ -191,7 +191,7 @@ jobs: sanitize-gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build executor: gcc10 - environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, BUILD_FLAGS: "", CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] sanitize-clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build @@ -201,13 +201,21 @@ jobs: threadsanitize-gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build executor: gcc10 - environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE_THREADS=ON, BUILD_FLAGS: "", CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] threadsanitize-clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build executor: clang10 environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] + nocheckeof-clang10: + description: Validate that when __SIMDJSON_CHECK_EOF=0, everything still succeeds + environment: + CXXFLAGS: -D__SIMDJSON_CHECK_EOF=0 + CMAKE_BUILD_FLAGS: --target ondemand_tests + CTEST_FLAGS: --output-on-failure -R ondemand_ + executor: clang10 + steps: [ cmake_test ] # dynamic dynamic-gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake dynamic build @@ -262,7 +270,7 @@ jobs: sanitize-haswell-gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build executor: gcc10 - environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, BUILD_FLAGS: "", CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] sanitize-haswell-clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build diff --git a/include/simdjson/common_defs.h b/include/simdjson/common_defs.h index aa8fe84bf..1a5abe9cd 100644 --- a/include/simdjson/common_defs.h +++ b/include/simdjson/common_defs.h @@ -251,6 +251,12 @@ namespace std { #endif #endif +// Feature flag for partially-implemented "don't require padding" feature +// TODO remove before 1.0 +#ifndef __SIMDJSON_CHECK_EOF +# define __SIMDJSON_CHECK_EOF 1 +#endif + #if SIMDJSON_CPLUSPLUS17 // if we have C++, then fallthrough is a default attribute # define simdjson_fallthrough [[fallthrough]] diff --git a/include/simdjson/generic/ondemand/json_iterator-inl.h b/include/simdjson/generic/ondemand/json_iterator-inl.h index ee8a2b5ef..a0094f060 100644 --- a/include/simdjson/generic/ondemand/json_iterator-inl.h +++ b/include/simdjson/generic/ondemand/json_iterator-inl.h @@ -186,10 +186,12 @@ simdjson_really_inline simdjson_result json_iterator::try_advan return json; } -simdjson_really_inline error_code json_iterator::require_tokens(uint32_t required_tokens) noexcept { +simdjson_really_inline error_code json_iterator::require_tokens(simdjson_unused uint32_t required_tokens) noexcept { +#if __SIMDJSON_CHECK_EOF if (position() + required_tokens > end_position()) { return report_error(TAPE_ERROR, "Document ended early"); } +#endif return SUCCESS; } diff --git a/tests/ondemand/ondemand_array_error_tests.cpp b/tests/ondemand/ondemand_array_error_tests.cpp index c95978623..9d5f652e8 100644 --- a/tests/ondemand/ondemand_array_error_tests.cpp +++ b/tests/ondemand/ondemand_array_error_tests.cpp @@ -89,13 +89,25 @@ namespace array_error_tests { bool array_iterate_unclosed_error() { TEST_START(); ONDEMAND_SUBTEST("unclosed extra comma", R"({ "a": [,)", assert_iterate(doc["a"], { INCORRECT_TYPE, TAPE_ERROR })); + #if __SIMDJSON_CHECK_EOF ONDEMAND_SUBTEST("unclosed extra comma", R"({ "a": [,,)", assert_iterate(doc["a"], { INCORRECT_TYPE, TAPE_ERROR })); + #else + ONDEMAND_SUBTEST("unclosed extra comma", R"({ "a": [,,)", assert_iterate(doc["a"], { INCORRECT_TYPE, INCORRECT_TYPE, TAPE_ERROR })); + #endif ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1 )", assert_iterate(doc["a"], { int64_t(1) }, { TAPE_ERROR })); // TODO These pass the user values that may run past the end of the buffer if they aren't careful // In particular, if the padding is decorated with the wrong values, we could cause overrun! + #if __SIMDJSON_CHECK_EOF ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1,)", assert_iterate(doc["a"], { int64_t(1) }, { TAPE_ERROR })); + #else + ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1,)", assert_iterate(doc["a"], { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR })); + #endif ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1)", assert_iterate(doc["a"], { NUMBER_ERROR, TAPE_ERROR })); + #if __SIMDJSON_CHECK_EOF ONDEMAND_SUBTEST("unclosed ", R"({ "a": [)", assert_iterate(doc["a"], { TAPE_ERROR })); + #else + ONDEMAND_SUBTEST("unclosed ", R"({ "a": [)", assert_iterate(doc["a"], { INCORRECT_TYPE, TAPE_ERROR })); + #endif TEST_SUCCEED(); } diff --git a/tests/ondemand/ondemand_object_error_tests.cpp b/tests/ondemand/ondemand_object_error_tests.cpp index e493ab922..4ddd6276d 100644 --- a/tests/ondemand/ondemand_object_error_tests.cpp +++ b/tests/ondemand/ondemand_object_error_tests.cpp @@ -79,7 +79,11 @@ namespace object_error_tests { } bool object_lookup_unclosed_error() { TEST_START(); + #if __SIMDJSON_CHECK_EOF ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_error(doc["a"], TAPE_ERROR)); + #else + ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_success(doc["a"])); + #endif ONDEMAND_SUBTEST("unclosed", R"({ "a" )", assert_error(doc["a"], TAPE_ERROR)); ONDEMAND_SUBTEST("unclosed", R"({ )", assert_error(doc["a"], TAPE_ERROR)); TEST_SUCCEED();