Compare commits

..

14 Commits

Author SHA1 Message Date
Daniel Lemire 41ea5eebff Silly compiler. 2023-03-13 12:50:46 -04:00
Daniel Lemire 63a4133db0 Merge branch 'master' into dlemire/issue1965 2023-03-13 12:30:12 -04:00
Daniel Lemire 7bf33f6c8a Bad compiler 2023-03-13 12:29:01 -04:00
Daniel Lemire d62b789cb3 More curly. 2023-03-13 10:49:15 -04:00
Daniel Lemire 63d384068e Update developer-options.cmake 2023-03-13 10:13:30 -04:00
Daniel Lemire 33dbd44098 Better way to put -Og in debug mode (#1967)
* Better way to put -Og in debug mode

* Silly compiler

* Stupid compiler.

* Stupid compiler.

* Silly compiler
2023-03-13 10:11:33 -04:00
Daniel Lemire 1469b6677b Unnecessary fixes 2023-03-13 09:46:07 -04:00
Daniel Lemire 4ca892ac57 Stupid compilers 2023-03-12 21:16:43 -04:00
Daniel Lemire e0dcf8adc9 Silly compiler 2023-03-12 20:57:45 -04:00
Daniel Lemire 13c7366817 Silly compiler 2023-03-10 22:04:56 -05:00
Daniel Lemire 296ae0b755 Stupid compiler 2023-03-09 14:09:50 -05:00
Daniel Lemire f6237ec0da Adding test. 2023-03-09 12:23:28 -05:00
Daniel Lemire 3e68899d30 Fix macro 2023-03-09 12:03:02 -05:00
Daniel Lemire 7c1412855c Might help with issue 1965 2023-03-09 11:58:11 -05:00
19 changed files with 123 additions and 49 deletions
+4 -4
View File
@@ -95,18 +95,18 @@ if(
)
endif()
# GCC and Clang have horrendous Debug builds when using SIMD.
# A common fix is to use '-Og' instead.
# bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412
if(
(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR
CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
AND CMAKE_BUILD_TYPE STREQUAL "Debug"
CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR
CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
)
message(STATUS "Adding -Og to compile flag")
simdjson_add_props(
target_compile_options PRIVATE
-Og
$<$<CONFIG:DEBUG>:-Og>
)
endif()
+17
View File
@@ -50,6 +50,23 @@ undefined behavior.")
endif()
endif()
option(SIMDJSON_SANITIZE_MEMORY "Sanitize memory" OFF)
if(SIMDJSON_SANITIZE_MEMORY)
message(STATUS "Setting the memory sanitizer.")
add_compile_options(
-fsanitize=memory -fno-sanitize-recover=all
)
link_libraries(
-fsanitize=memory -fno-sanitize-recover=all
)
# Ubuntu bug for GCC 5.0+ (safe for all versions)
if(CMAKE_COMPILER_IS_GNUCC)
link_libraries(-fuse-ld=gold)
endif()
endif()
if(SIMDJSON_SANITIZE_THREADS)
message(STATUS "Setting both the thread sanitizer \
and the undefined-behavior sanitizer.")
+4
View File
@@ -9,6 +9,10 @@ namespace {
// but the algorithms do not end up using the returned value.
// Sadly, sanitizers are not smart enough to figure it out.
SIMDJSON_NO_SANITIZE_UNDEFINED
// This function can be used safely even if not all bytes have been
// initialized.
// See issue https://github.com/simdjson/simdjson/issues/1965
SIMDJSON_NO_SANITIZE_MEMORY
simdjson_inline int trailing_zeroes(uint64_t input_num) {
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
unsigned long ret;
@@ -9,6 +9,10 @@ namespace {
// but the algorithms do not end up using the returned value.
// Sadly, sanitizers are not smart enough to figure it out.
SIMDJSON_NO_SANITIZE_UNDEFINED
// This function can be used safely even if not all bytes have been
// initialized.
// See issue https://github.com/simdjson/simdjson/issues/1965
SIMDJSON_NO_SANITIZE_MEMORY
simdjson_inline int trailing_zeroes(uint64_t input_num) {
#if SIMDJSON_REGULAR_VISUAL_STUDIO
return (int)_tzcnt_u64(input_num);
@@ -9,6 +9,10 @@ namespace {
// but the algorithms do not end up using the returned value.
// Sadly, sanitizers are not smart enough to figure it out.
SIMDJSON_NO_SANITIZE_UNDEFINED
// This function can be used safely even if not all bytes have been
// initialized.
// See issue https://github.com/simdjson/simdjson/issues/1965
SIMDJSON_NO_SANITIZE_MEMORY
simdjson_inline int trailing_zeroes(uint64_t input_num) {
#if SIMDJSON_REGULAR_VISUAL_STUDIO
return (int)_tzcnt_u64(input_num);
+13
View File
@@ -148,6 +148,19 @@ use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
#define SIMDJSON_NO_SANITIZE_UNDEFINED
#endif
#if defined(__clang__) || defined(__GNUC__)
#if defined(__has_feature)
# if __has_feature(memory_sanitizer)
#define SIMDJSON_NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory")))
# endif // if __has_feature(memory_sanitizer)
#endif // defined(__has_feature)
#endif
// make sure it is defined as 'nothing' if it is unapplicable.
#ifndef SIMDJSON_NO_SANITIZE_MEMORY
#define SIMDJSON_NO_SANITIZE_MEMORY
#endif
#if SIMDJSON_VISUAL_STUDIO
// This is one case where we do not distinguish between
// regular visual studio and clang under visual studio.
+4
View File
@@ -9,6 +9,10 @@ namespace {
// but the algorithms do not end up using the returned value.
// Sadly, sanitizers are not smart enough to figure it out.
SIMDJSON_NO_SANITIZE_UNDEFINED
// This function can be used safely even if not all bytes have been
// initialized.
// See issue https://github.com/simdjson/simdjson/issues/1965
SIMDJSON_NO_SANITIZE_MEMORY
simdjson_inline int trailing_zeroes(uint64_t input_num) {
#if SIMDJSON_REGULAR_VISUAL_STUDIO
unsigned long ret;
@@ -9,6 +9,10 @@ namespace {
// but the algorithms do not end up using the returned value.
// Sadly, sanitizers are not smart enough to figure it out.
SIMDJSON_NO_SANITIZE_UNDEFINED
// This function can be used safely even if not all bytes have been
// initialized.
// See issue https://github.com/simdjson/simdjson/issues/1965
SIMDJSON_NO_SANITIZE_MEMORY
simdjson_inline int trailing_zeroes(uint64_t input_num) {
#if SIMDJSON_REGULAR_VISUAL_STUDIO
unsigned long ret;
+3 -3
View File
@@ -634,7 +634,7 @@ namespace document_stream_tests {
ASSERT_SUCCESS( odparser.parse_many(json.data(), json.length(), 50).get(odstream) );
for (auto doc: odstream) {
if(counter < 6) {
int64_t val;
int64_t val{};
ASSERT_SUCCESS(doc.at_pointer("/4").get(val));
ASSERT_EQUAL(val, 5);
} else {
@@ -797,7 +797,7 @@ namespace document_stream_tests {
simdjson::dom::document_stream stream;
ASSERT_SUCCESS( parser.parse_many(str, batch_size).get(stream) );
for (auto doc : stream) {
int64_t keyid;
int64_t keyid{};
ASSERT_SUCCESS( doc["id"].get(keyid) );
ASSERT_EQUAL( keyid, int64_t(count) );
@@ -837,7 +837,7 @@ namespace document_stream_tests {
simdjson::dom::document_stream stream;
ASSERT_SUCCESS( parser.parse_many(str, batch_size).get(stream) );
for (auto doc : stream) {
int64_t keyid;
int64_t keyid{};
ASSERT_SUCCESS( doc["id"].get(keyid) );
ASSERT_EQUAL( keyid, int64_t(count) );
+2 -2
View File
@@ -46,7 +46,7 @@ namespace parser_load {
ASSERT_SUCCESS(parser.parse_many(DOC).get(docs));
for (auto doc : docs) {
count++;
uint64_t val;
uint64_t val{};
auto error = doc.get(val);
if (count == 3) {
ASSERT_ERROR(error, TAPE_ERROR);
@@ -83,7 +83,7 @@ namespace parser_load {
ASSERT_SUCCESS(parser.parse_many(DOC).get(docs));
for (auto doc : docs) {
count++;
uint64_t val;
uint64_t val{};
auto error = doc.get(val);
if (count == 3) {
ASSERT_ERROR(error, TAPE_ERROR);
+2 -2
View File
@@ -33,12 +33,12 @@ static bool parse_and_validate(const std::string src, T expected) {
simdjson::dom::parser parser;
if constexpr (std::is_same<int64_t, T>::value) {
int64_t actual;
int64_t actual{};
ASSERT_SUCCESS( parser.parse(pstr)["key"].get(actual) );
std::cout << std::boolalpha << "test: " << (expected == actual) << std::endl;
ASSERT_EQUAL( expected, actual );
} else {
uint64_t actual;
uint64_t actual{};
ASSERT_SUCCESS( parser.parse(pstr)["key"].get(actual) );
std::cout << std::boolalpha << "test: " << (expected == actual) << std::endl;
ASSERT_EQUAL( expected, actual );
+17 -17
View File
@@ -134,7 +134,7 @@ namespace array_tests {
ondemand::parser parser;
ondemand::document doc;
ASSERT_SUCCESS(parser.iterate(json).get(doc));
size_t count;
size_t count{};
ASSERT_SUCCESS(doc.count_elements().get(count));
ondemand::array arr;
ASSERT_SUCCESS(doc.get_array().get(arr));
@@ -182,7 +182,7 @@ namespace array_tests {
ASSERT_SUCCESS(doc.get_object().get(obj));
ondemand::value v;
ASSERT_SUCCESS(doc.find_field("test").get(v));
size_t count;
size_t count{};
ASSERT_SUCCESS(v.count_elements().get(count));
ASSERT_EQUAL(count, 3);
ASSERT_SUCCESS(doc.find_field("joe").get(v));
@@ -200,7 +200,7 @@ namespace array_tests {
ondemand::array array;
ASSERT_RESULT( doc_result.type(), json_type::array );
ASSERT_SUCCESS( doc_result.get_array().get(array) );
size_t count;
size_t count{};
ASSERT_SUCCESS( array.count_elements().get(count) );
ASSERT_EQUAL(count, expected_value.size());
return true;
@@ -209,13 +209,13 @@ namespace array_tests {
ondemand::array array;
ASSERT_RESULT( doc_result.type(), json_type::array );
ASSERT_SUCCESS( doc_result.get(array) );
size_t count;
size_t count{};
ASSERT_SUCCESS( array.count_elements().get(count) );
ASSERT_EQUAL(count, expected_value.size());
size_t i = 0;
std::vector<uint64_t> receiver(count);
for (auto value : array) {
uint64_t actual;
uint64_t actual{};
ASSERT_SUCCESS( value.get(actual) );
ASSERT_EQUAL(actual, expected_value[i]);
receiver[i] = actual;
@@ -235,7 +235,7 @@ namespace array_tests {
ondemand::array array;
ASSERT_RESULT( doc_result.type(), json_type::array );
ASSERT_SUCCESS( doc_result.get_array().get(array) );
size_t count;
size_t count{};
ASSERT_SUCCESS( array.count_elements().get(count) );
ASSERT_EQUAL(count, 0);
return true;
@@ -244,13 +244,13 @@ namespace array_tests {
ondemand::array array;
ASSERT_RESULT( doc_result.type(), json_type::array );
ASSERT_SUCCESS( doc_result.get(array) );
size_t count;
size_t count{};
ASSERT_SUCCESS( array.count_elements().get(count) );
ASSERT_EQUAL(count, 0);
size_t i = 0;
std::vector<uint64_t> receiver(count);
for (auto value : array) {
uint64_t actual;
uint64_t actual{};
ASSERT_SUCCESS( value.get(actual) );
i++;
}
@@ -269,7 +269,7 @@ namespace array_tests {
ondemand::array array;
ASSERT_RESULT( doc_result.type(), json_type::array );
ASSERT_SUCCESS( doc_result.get(array) );
size_t count;
size_t count{};
auto e = array.count_elements().get(count);
if( e != TAPE_ERROR) {
std::cout << e << "\n";
@@ -285,7 +285,7 @@ namespace array_tests {
TEST_START();
auto empty = R"( [] )"_padded;
SUBTEST("ondemand::empty_doc_array", test_ondemand_doc(empty, [&](auto doc_result) {
size_t count;
size_t count{};
ASSERT_RESULT( doc_result.type(), json_type::array );
ASSERT_SUCCESS( doc_result.count_elements().get(count) );
ASSERT_EQUAL( count, 0 );
@@ -293,7 +293,7 @@ namespace array_tests {
}));
auto basic = R"( [-1.234, 100000000000000, null, [1,2,3], {"t":true, "f":false}] )"_padded;
SUBTEST("ondemand::basic_doc_array", test_ondemand_doc(basic, [&](auto doc_result) {
size_t count;
size_t count{};
ASSERT_RESULT( doc_result.type(), json_type::array );
ASSERT_SUCCESS( doc_result.count_elements().get(count) );
ASSERT_EQUAL( count, 5 );
@@ -331,7 +331,7 @@ namespace array_tests {
size_t i = 0;
for (auto value : array) {
int64_t actual;
int64_t actual{};
ASSERT_SUCCESS( value.get(actual) );
ASSERT_EQUAL(actual, expected_value[i]);
i++;
@@ -355,7 +355,7 @@ namespace array_tests {
ASSERT_SUCCESS( doc_result.get(array) );
i = 0;
for (auto value : array) {
int64_t actual;
int64_t actual{};
ASSERT_SUCCESS( value.get(actual) );
container[i] = actual;
i++;
@@ -379,7 +379,7 @@ namespace array_tests {
array.reset();
i = 0;
for (auto value : array) {
int64_t actual;
int64_t actual{};
ASSERT_SUCCESS( value.get(actual) );
container[i] = actual;
i++;
@@ -435,7 +435,7 @@ namespace array_tests {
TEST_SUCCEED();
}
bool count_empty(simdjson::ondemand::array arr) {
size_t count;
size_t count{};
ASSERT_SUCCESS(arr.count_elements().get(count));
ASSERT_EQUAL(count, 0);
bool is_empty;
@@ -466,7 +466,7 @@ namespace array_tests {
for (auto d : data) {
simdjson::ondemand::array arr;
ASSERT_SUCCESS(d.get_array().get(arr));
size_t count;
size_t count{};
ASSERT_SUCCESS(arr.count_elements().get(count));
ASSERT_EQUAL(count, 4);
}
@@ -528,7 +528,7 @@ namespace array_tests {
size_t i=0;
for (auto value : array) {
int64_t actual;
int64_t actual{};
ASSERT_SUCCESS( value.get(actual) );
ASSERT_EQUAL(actual, expected_value[i]);
i++;
@@ -7,7 +7,7 @@ namespace document_stream_tests {
template <typename T>
bool process_doc(T &docref) {
int64_t val;
int64_t val{};
ASSERT_SUCCESS(docref.at_pointer("/4").get(val));
ASSERT_EQUAL(val, 5);
return true;
@@ -567,7 +567,7 @@ namespace document_stream_tests {
size_t count{0};
ASSERT_SUCCESS( parser.iterate_many(str, batch_size).get(stream) );
for (auto doc : stream) {
int64_t keyid;
int64_t keyid{};
ASSERT_SUCCESS( doc["id"].get(keyid) );
ASSERT_EQUAL( keyid, int64_t(count) );
@@ -604,7 +604,7 @@ namespace document_stream_tests {
ASSERT_SUCCESS( odparser.iterate_many(json.data(), json.length(), 50).get(odstream) );
for (auto doc: odstream) {
if(counter < 6) {
int64_t val;
int64_t val{};
ASSERT_SUCCESS(doc.at_pointer("/4").get(val));
ASSERT_EQUAL(val, 5);
} else {
@@ -643,7 +643,7 @@ namespace document_stream_tests {
size_t count{0};
ASSERT_SUCCESS( parser.iterate_many(str, batch_size).get(stream) );
for (auto doc : stream) {
int64_t keyid;
int64_t keyid{};
ASSERT_SUCCESS( doc["id"].get(keyid) );
ASSERT_EQUAL( keyid, int64_t(count) );
+1 -1
View File
@@ -67,7 +67,7 @@ namespace misc_tests {
ASSERT_SUCCESS(val.get_object().get(obj));
ondemand::array arr;
ASSERT_SUCCESS(obj["a"].get_array().get(arr));
size_t count;
size_t count{};
ASSERT_SUCCESS(arr.count_elements().get(count));
ASSERT_EQUAL(3,count);
TEST_SUCCEED();
+2 -2
View File
@@ -238,7 +238,7 @@ namespace number_tests {
ondemand::number num;
ASSERT_SUCCESS(val.get_number().get(num));
ASSERT_EQUAL(is_negative[counter], val.is_negative());
bool intvalue;
bool intvalue{};
ASSERT_SUCCESS(val.is_integer().get(intvalue));
ASSERT_EQUAL(is_integer[counter], intvalue);
ondemand::number_type t = num.get_number_type();
@@ -341,7 +341,7 @@ namespace number_tests {
ondemand::number number;
ondemand::number_type nt;
bool intvalue;
bool intvalue{};
docdata = R"(1.0)"_padded;
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
+31 -8
View File
@@ -6,6 +6,7 @@ using namespace simdjson;
namespace object_tests {
using namespace std;
using simdjson::ondemand::json_type;
bool issue1745() {
TEST_START();
auto json = R"({
@@ -225,6 +226,25 @@ namespace object_tests {
}
#if SIMDJSON_EXCEPTIONS
bool issue1965() {
TEST_START();
std::string str = "{\"query\":\"ah\"}";
std::unique_ptr<char[]> buffer(new char[str.size() + simdjson::SIMDJSON_PADDING]);
memcpy(buffer.get(), str.data(), str.size());
simdjson::padded_string_view view(buffer.get(), str.size(), str.size() + simdjson::SIMDJSON_PADDING);
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(view);
simdjson::ondemand::object root = doc.get_object();
simdjson::ondemand::value query = root.find_field("query");
simdjson::ondemand::raw_json_string raw = query.get_raw_json_string();
std::unique_ptr<uint8_t[]> dst_buffer(new uint8_t[3 + simdjson::SIMDJSON_PADDING]);
uint8_t * dst = dst_buffer.get();
std::string_view fieldstring = parser.unescape(raw, dst);
std::cout << fieldstring << std::endl;
TEST_SUCCEED();
}
bool issue1745_with_exceptions() {
TEST_START();
auto json = R"({
@@ -889,7 +909,7 @@ namespace object_tests {
ondemand::document doc;
ASSERT_SUCCESS(parser.iterate(json).get(doc));
ondemand::object obj;
size_t count;
size_t count{};
ASSERT_SUCCESS(doc.get_object().get(obj));
ASSERT_SUCCESS(obj.count_fields().get(count));
ASSERT_EQUAL(count, 0);
@@ -904,7 +924,7 @@ namespace object_tests {
ASSERT_SUCCESS(parser.iterate(json).get(doc));
ondemand::object obj;
ASSERT_SUCCESS(doc.get_object().get(obj));
size_t count;
size_t count{};
ASSERT_SUCCESS(obj.count_fields().get(count));
ASSERT_EQUAL(count, 0);
for (auto field : obj) {
@@ -923,7 +943,7 @@ namespace object_tests {
ondemand::parser parser;
ondemand::document doc;
ASSERT_SUCCESS(parser.iterate(json).get(doc));
size_t count;
size_t count{};
ASSERT_SUCCESS(doc.count_fields().get(count));
ondemand::object obj;
ASSERT_SUCCESS(doc.get_object().get(obj));
@@ -969,7 +989,7 @@ namespace object_tests {
for (auto d : data) {
simdjson::ondemand::object obj;
ASSERT_SUCCESS(d.get_object().get(obj));
size_t count;
size_t count{};
ASSERT_SUCCESS(obj.count_fields().get(count));
ASSERT_EQUAL(count, 7);
}
@@ -1005,7 +1025,7 @@ namespace object_tests {
simdjson::ondemand::array data;
ASSERT_SUCCESS(doc["result"]["data"].get_array().get(data));
for (auto d : data) {
size_t count;
size_t count{};
ASSERT_SUCCESS(d.count_fields().get(count));
ASSERT_EQUAL(count, 7);
}
@@ -1019,7 +1039,7 @@ namespace object_tests {
ondemand::document doc;
ASSERT_SUCCESS(parser.iterate(json).get(doc));
ondemand::object obj;
size_t count;
size_t count{};
ASSERT_SUCCESS(doc.get_object().get(obj));
ASSERT_SUCCESS(obj.count_fields().get(count));
ASSERT_EQUAL(count, 5);
@@ -1081,7 +1101,7 @@ namespace object_tests {
TEST_START();
auto empty = R"( {} )"_padded;
SUBTEST("ondemand::empty_doc_object", test_ondemand_doc(empty, [&](auto doc_result) {
size_t count;
size_t count{};
ASSERT_RESULT( doc_result.type(), json_type::object );
ASSERT_SUCCESS( doc_result.count_fields().get(count) );
ASSERT_EQUAL( count, 0 );
@@ -1089,7 +1109,7 @@ namespace object_tests {
}));
auto basic = R"( {"a":-1.234, "b":false, "c":null, "d":[1000.1,-2000.2,3000.3], "e":{"a":true, "b":false}} )"_padded;
SUBTEST("ondemand::basic_doc_object", test_ondemand_doc(basic, [&](auto doc_result) {
size_t count;
size_t count{};
ASSERT_RESULT( doc_result.type(), json_type::object );
ASSERT_SUCCESS( doc_result.count_fields().get(count) );
ASSERT_EQUAL( count, 5 );
@@ -1221,6 +1241,9 @@ namespace object_tests {
bool run() {
return
#if SIMDJSON_EXCEPTIONS
issue1965() &&
#endif
issue1876a() &&
issue1876() &&
test_strager() &&
+3 -3
View File
@@ -320,7 +320,7 @@ bool json_object_count() {
ondemand::parser parser;
ondemand::document doc;
ASSERT_SUCCESS(parser.iterate(json).get(doc));
size_t count;
size_t count{};
ASSERT_SUCCESS(doc.count_fields().get(count));
ASSERT_EQUAL(count,1);
ondemand::object object;
@@ -778,9 +778,9 @@ bool ndjson_basics_example() {
size_t count{0};
int64_t expected[3] = {1,2,3};
for (auto doc : docs) {
int64_t actual;
int64_t actual{};
ASSERT_SUCCESS( doc["foo"].get(actual) );
ASSERT_EQUAL( actual,expected[count++] );
ASSERT_EQUAL( actual, expected[count++] );
}
TEST_SUCCEED();
}
+3 -2
View File
@@ -79,7 +79,7 @@ namespace twitter_tests {
auto media = tweet["entities"]["media"];
if (!media.error()) {
for (auto image : media) {
uint64_t id_val;
uint64_t id_val{};
std::string_view id_string;
ASSERT_SUCCESS( image["id"].get(id_val) );
ASSERT_SUCCESS( image["id_str"].get(id_string) );
@@ -91,7 +91,8 @@ namespace twitter_tests {
ASSERT_SUCCESS( size.unescaped_key().get(size_key) );
std::cout << "Type of image size = " << size_key << std::endl;
uint64_t width, height;
uint64_t width{};
uint64_t height{};
ASSERT_SUCCESS( size.value()["w"].get(width) );
ASSERT_SUCCESS( size.value()["h"].get(height) );
image_sizes.insert(make_pair(width, height));
+1 -1
View File
@@ -10,7 +10,7 @@
template<typename T, typename F>
bool test_ondemand(simdjson::ondemand::parser &parser, const simdjson::padded_string &json, const F& f) {
auto doc = parser.iterate(json);
T val;
T val{};
ASSERT_SUCCESS( doc.get(val) );
return f(val);
}