admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add __SIMDJSON_CHECK_EOF feature flag

Browse Source
This commit is contained in:
John Keiser
2021-03-22 11:03:00 -07:00
committed by John Keiser
parent 78d1cd96a9
commit 90409897eb
5 changed files with 42 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+17 -9
View File
@@ -11,7 +11,7 @@ executors:
environment:
CXX: g++-8
CC: gcc-8
BUILD_FLAGS:
CMAKE_BUILD_FLAGS:
CTEST_FLAGS: --output-on-failure
gcc9:
@@ -20,7 +20,7 @@ executors:
environment:
CXX: g++-9
CC: gcc-9
BUILD_FLAGS:
CMAKE_BUILD_FLAGS:
CTEST_FLAGS: --output-on-failure
gcc10:
@@ -29,7 +29,7 @@ executors:
environment:
CXX: g++-10
CC: gcc-10
BUILD_FLAGS:
CMAKE_BUILD_FLAGS:
CTEST_FLAGS: --output-on-failure
clang10:
@@ -38,7 +38,7 @@ executors:
environment:
CXX: clang++-10
CC: clang-10
BUILD_FLAGS:
CMAKE_BUILD_FLAGS:
CTEST_FLAGS: --output-on-failure
clang9:
@@ -47,7 +47,7 @@ executors:
environment:
CXX: clang++-9
CC: clang-9
BUILD_FLAGS:
CMAKE_BUILD_FLAGS:
CTEST_FLAGS: --output-on-failure
clang6:
@@ -56,7 +56,7 @@ executors:
environment:
CXX: clang++-6.0
CC: clang-6.0
BUILD_FLAGS:
CMAKE_BUILD_FLAGS:
CTEST_FLAGS: --output-on-failure
# Reusable test commands (and initializer for clang 6)
@@ -191,7 +191,7 @@ jobs:
sanitize-gcc10:
description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
executor: gcc10
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, BUILD_FLAGS: "", CTEST_FLAGS: --output-on-failure -LE explicitonly }
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
steps: [ cmake_test ]
sanitize-clang10:
description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
@@ -201,13 +201,21 @@ jobs:
threadsanitize-gcc10:
description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
executor: gcc10
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE_THREADS=ON, BUILD_FLAGS: "", CTEST_FLAGS: --output-on-failure -LE explicitonly }
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
steps: [ cmake_test ]
threadsanitize-clang10:
description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
executor: clang10
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
steps: [ cmake_test ]
nocheckeof-clang10:
description: Validate that when __SIMDJSON_CHECK_EOF=0, everything still succeeds
environment:
CXXFLAGS: -D__SIMDJSON_CHECK_EOF=0
CMAKE_BUILD_FLAGS: --target ondemand_tests
CTEST_FLAGS: --output-on-failure -R ondemand_
executor: clang10
steps: [ cmake_test ]
# dynamic
dynamic-gcc10:
description: Build and run tests on GCC 10 and AVX 2 with a cmake dynamic build
@@ -262,7 +270,7 @@ jobs:
sanitize-haswell-gcc10:
description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
executor: gcc10
environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, BUILD_FLAGS: "", CTEST_FLAGS: --output-on-failure -LE explicitonly }
environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
steps: [ cmake_test ]
sanitize-haswell-clang10:
description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
include/simdjson/common_defs.h
+6
View File
@@ -251,6 +251,12 @@ namespace std {
#endif
#endif
// Feature flag for partially-implemented "don't require padding" feature
// TODO remove before 1.0
#ifndef __SIMDJSON_CHECK_EOF
# define __SIMDJSON_CHECK_EOF 1
#endif
#if SIMDJSON_CPLUSPLUS17
// if we have C++, then fallthrough is a default attribute
# define simdjson_fallthrough [[fallthrough]]
include/simdjson/generic/ondemand/json_iterator-inl.h
+3 -1
View File
@@ -186,10 +186,12 @@ simdjson_really_inline simdjson_result<const uint8_t *> json_iterator::try_advan
return json;
}
simdjson_really_inline error_code json_iterator::require_tokens(uint32_t required_tokens) noexcept {
simdjson_really_inline error_code json_iterator::require_tokens(simdjson_unused uint32_t required_tokens) noexcept {
#if __SIMDJSON_CHECK_EOF
if (position() + required_tokens > end_position()) {
return report_error(TAPE_ERROR, "Document ended early");
}
#endif
return SUCCESS;
}
tests/ondemand/ondemand_array_error_tests.cpp
+12
View File
@@ -89,13 +89,25 @@ namespace array_error_tests {
bool array_iterate_unclosed_error() {
TEST_START();
ONDEMAND_SUBTEST("unclosed extra comma", R"({ "a": [,)", assert_iterate(doc["a"], { INCORRECT_TYPE, TAPE_ERROR }));
#if __SIMDJSON_CHECK_EOF
ONDEMAND_SUBTEST("unclosed extra comma", R"({ "a": [,,)", assert_iterate(doc["a"], { INCORRECT_TYPE, TAPE_ERROR }));
#else
ONDEMAND_SUBTEST("unclosed extra comma", R"({ "a": [,,)", assert_iterate(doc["a"], { INCORRECT_TYPE, INCORRECT_TYPE, TAPE_ERROR }));
#endif
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1 )", assert_iterate(doc["a"], { int64_t(1) }, { TAPE_ERROR }));
// TODO These pass the user values that may run past the end of the buffer if they aren't careful
// In particular, if the padding is decorated with the wrong values, we could cause overrun!
#if __SIMDJSON_CHECK_EOF
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1,)", assert_iterate(doc["a"], { int64_t(1) }, { TAPE_ERROR }));
#else
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1,)", assert_iterate(doc["a"], { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR }));
#endif
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1)", assert_iterate(doc["a"], { NUMBER_ERROR, TAPE_ERROR }));
#if __SIMDJSON_CHECK_EOF
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [)", assert_iterate(doc["a"], { TAPE_ERROR }));
#else
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [)", assert_iterate(doc["a"], { INCORRECT_TYPE, TAPE_ERROR }));
#endif
TEST_SUCCEED();
}
tests/ondemand/ondemand_object_error_tests.cpp
+4
View File
@@ -79,7 +79,11 @@ namespace object_error_tests {
}
bool object_lookup_unclosed_error() {
TEST_START();
#if __SIMDJSON_CHECK_EOF
ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_error(doc["a"], TAPE_ERROR));
#else
ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_success(doc["a"]));
#endif
ONDEMAND_SUBTEST("unclosed", R"({ "a" )", assert_error(doc["a"], TAPE_ERROR));
ONDEMAND_SUBTEST("unclosed", R"({ )", assert_error(doc["a"], TAPE_ERROR));
TEST_SUCCEED();