Silly compiler.

cmake/developer-options.cmake

@@ -50,6 +50,23 @@ undefined behavior.")
   endif()
 endif()
 
+option(SIMDJSON_SANITIZE_MEMORY "Sanitize memory" OFF)
+
+
+if(SIMDJSON_SANITIZE_MEMORY)
+  message(STATUS "Setting the memory sanitizer.")
+  add_compile_options(
+      -fsanitize=memory -fno-sanitize-recover=all
+  )
+  link_libraries(
+      -fsanitize=memory -fno-sanitize-recover=all
+  )
+  # Ubuntu bug for GCC 5.0+ (safe for all versions)
+  if(CMAKE_COMPILER_IS_GNUCC)
+    link_libraries(-fuse-ld=gold)
+  endif()
+endif()
+
 if(SIMDJSON_SANITIZE_THREADS)
   message(STATUS "Setting both the thread sanitizer \
 and the undefined-behavior sanitizer.")

include/simdjson/arm64/bitmanipulation.h

@@ -9,6 +9,10 @@ namespace {
 // but the algorithms do not end up using the returned value.
 // Sadly, sanitizers are not smart enough to figure it out.
 SIMDJSON_NO_SANITIZE_UNDEFINED
+// This function can be used safely even if not all bytes have been
+// initialized.
+// See issue https://github.com/simdjson/simdjson/issues/1965
+SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_inline int trailing_zeroes(uint64_t input_num) {
 #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
   unsigned long ret;

include/simdjson/haswell/bitmanipulation.h

@@ -9,6 +9,10 @@ namespace {
 // but the algorithms do not end up using the returned value.
 // Sadly, sanitizers are not smart enough to figure it out.
 SIMDJSON_NO_SANITIZE_UNDEFINED
+// This function can be used safely even if not all bytes have been
+// initialized.
+// See issue https://github.com/simdjson/simdjson/issues/1965
+SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_inline int trailing_zeroes(uint64_t input_num) {
 #if SIMDJSON_REGULAR_VISUAL_STUDIO
   return (int)_tzcnt_u64(input_num);

include/simdjson/icelake/bitmanipulation.h

@@ -9,6 +9,10 @@ namespace {
 // but the algorithms do not end up using the returned value.
 // Sadly, sanitizers are not smart enough to figure it out.
 SIMDJSON_NO_SANITIZE_UNDEFINED
+// This function can be used safely even if not all bytes have been
+// initialized.
+// See issue https://github.com/simdjson/simdjson/issues/1965
+SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_inline int trailing_zeroes(uint64_t input_num) {
 #if SIMDJSON_REGULAR_VISUAL_STUDIO
   return (int)_tzcnt_u64(input_num);

include/simdjson/portability.h

@@ -148,6 +148,19 @@ use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
 #define SIMDJSON_NO_SANITIZE_UNDEFINED
 #endif
 
+
+#if defined(__clang__) || defined(__GNUC__)
+#if defined(__has_feature)
+#  if __has_feature(memory_sanitizer)
+#define SIMDJSON_NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory")))
+#  endif // if __has_feature(memory_sanitizer)
+#endif // defined(__has_feature)
+#endif
+// make sure it is defined as 'nothing' if it is unapplicable.
+#ifndef SIMDJSON_NO_SANITIZE_MEMORY
+#define SIMDJSON_NO_SANITIZE_MEMORY
+#endif
+
 #if SIMDJSON_VISUAL_STUDIO
 // This is one case where we do not distinguish between
 // regular visual studio and clang under visual studio.

include/simdjson/ppc64/bitmanipulation.h

@@ -9,6 +9,10 @@ namespace {
 // but the algorithms do not end up using the returned value.
 // Sadly, sanitizers are not smart enough to figure it out.
 SIMDJSON_NO_SANITIZE_UNDEFINED
+// This function can be used safely even if not all bytes have been
+// initialized.
+// See issue https://github.com/simdjson/simdjson/issues/1965
+SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_inline int trailing_zeroes(uint64_t input_num) {
 #if SIMDJSON_REGULAR_VISUAL_STUDIO
   unsigned long ret;

include/simdjson/westmere/bitmanipulation.h

@@ -9,6 +9,10 @@ namespace {
 // but the algorithms do not end up using the returned value.
 // Sadly, sanitizers are not smart enough to figure it out.
 SIMDJSON_NO_SANITIZE_UNDEFINED
+// This function can be used safely even if not all bytes have been
+// initialized.
+// See issue https://github.com/simdjson/simdjson/issues/1965
+SIMDJSON_NO_SANITIZE_MEMORY
 simdjson_inline int trailing_zeroes(uint64_t input_num) {
 #if SIMDJSON_REGULAR_VISUAL_STUDIO
   unsigned long ret;

tests/dom/document_stream_tests.cpp

@@ -634,7 +634,7 @@ namespace document_stream_tests {
     ASSERT_SUCCESS( odparser.parse_many(json.data(), json.length(), 50).get(odstream) );
     for (auto doc: odstream) {
       if(counter < 6) {
-        int64_t val;
+        int64_t val{};
         ASSERT_SUCCESS(doc.at_pointer("/4").get(val));
         ASSERT_EQUAL(val, 5);
       } else {

tests/ondemand/ondemand_array_tests.cpp

@@ -134,7 +134,7 @@ namespace array_tests {
     ondemand::parser parser;
     ondemand::document doc;
     ASSERT_SUCCESS(parser.iterate(json).get(doc));
-    size_t count;
+    size_t count{};
     ASSERT_SUCCESS(doc.count_elements().get(count));
     ondemand::array arr;
     ASSERT_SUCCESS(doc.get_array().get(arr));
@@ -182,7 +182,7 @@ namespace array_tests {
     ASSERT_SUCCESS(doc.get_object().get(obj));
     ondemand::value v;
     ASSERT_SUCCESS(doc.find_field("test").get(v));
-    size_t count;
+    size_t count{};
     ASSERT_SUCCESS(v.count_elements().get(count));
     ASSERT_EQUAL(count, 3);
     ASSERT_SUCCESS(doc.find_field("joe").get(v));
@@ -200,7 +200,7 @@ namespace array_tests {
       ondemand::array array;
       ASSERT_RESULT( doc_result.type(), json_type::array );
       ASSERT_SUCCESS( doc_result.get_array().get(array) );
-      size_t count;
+      size_t count{};
       ASSERT_SUCCESS( array.count_elements().get(count) );
       ASSERT_EQUAL(count, expected_value.size());
       return true;
@@ -209,7 +209,7 @@ namespace array_tests {
       ondemand::array array;
       ASSERT_RESULT( doc_result.type(), json_type::array );
       ASSERT_SUCCESS( doc_result.get(array) );
-      size_t count;
+      size_t count{};
       ASSERT_SUCCESS( array.count_elements().get(count) );
       ASSERT_EQUAL(count, expected_value.size());
       size_t i = 0;
@@ -235,7 +235,7 @@ namespace array_tests {
       ondemand::array array;
       ASSERT_RESULT( doc_result.type(), json_type::array );
       ASSERT_SUCCESS( doc_result.get_array().get(array) );
-      size_t count;
+      size_t count{};
       ASSERT_SUCCESS( array.count_elements().get(count) );
       ASSERT_EQUAL(count, 0);
       return true;
@@ -244,7 +244,7 @@ namespace array_tests {
       ondemand::array array;
       ASSERT_RESULT( doc_result.type(), json_type::array );
       ASSERT_SUCCESS( doc_result.get(array) );
-      size_t count;
+      size_t count{};
       ASSERT_SUCCESS( array.count_elements().get(count) );
       ASSERT_EQUAL(count, 0);
       size_t i = 0;
@@ -269,7 +269,7 @@ namespace array_tests {
       ondemand::array array;
       ASSERT_RESULT( doc_result.type(), json_type::array );
       ASSERT_SUCCESS( doc_result.get(array) );
-      size_t count;
+      size_t count{};
       auto e = array.count_elements().get(count);
       if( e != TAPE_ERROR) {
         std::cout << e << "\n";
@@ -285,7 +285,7 @@ namespace array_tests {
     TEST_START();
     auto empty = R"( [] )"_padded;
     SUBTEST("ondemand::empty_doc_array", test_ondemand_doc(empty, [&](auto doc_result) {
-        size_t count;
+        size_t count{};
         ASSERT_RESULT( doc_result.type(), json_type::array );
         ASSERT_SUCCESS( doc_result.count_elements().get(count) );
         ASSERT_EQUAL( count, 0 );
@@ -293,7 +293,7 @@ namespace array_tests {
     }));
     auto basic = R"( [-1.234, 100000000000000, null, [1,2,3], {"t":true, "f":false}] )"_padded;
     SUBTEST("ondemand::basic_doc_array", test_ondemand_doc(basic, [&](auto doc_result) {
-        size_t count;
+        size_t count{};
         ASSERT_RESULT( doc_result.type(), json_type::array );
         ASSERT_SUCCESS( doc_result.count_elements().get(count) );
         ASSERT_EQUAL( count, 5 );
@@ -435,7 +435,7 @@ namespace array_tests {
     TEST_SUCCEED();
   }
   bool count_empty(simdjson::ondemand::array arr) {
-    size_t count;
+    size_t count{};
     ASSERT_SUCCESS(arr.count_elements().get(count));
     ASSERT_EQUAL(count, 0);
     bool is_empty;
@@ -466,7 +466,7 @@ namespace array_tests {
     for (auto d : data) {
       simdjson::ondemand::array arr;
       ASSERT_SUCCESS(d.get_array().get(arr));
-      size_t count;
+      size_t count{};
       ASSERT_SUCCESS(arr.count_elements().get(count));
       ASSERT_EQUAL(count, 4);
     }

tests/ondemand/ondemand_document_stream_tests.cpp

@@ -7,7 +7,7 @@ namespace document_stream_tests {
 
     template <typename T>
     bool process_doc(T &docref) {
-        int64_t val;
+        int64_t val{};
         ASSERT_SUCCESS(docref.at_pointer("/4").get(val));
         ASSERT_EQUAL(val, 5);
         return true;
@@ -604,7 +604,7 @@ namespace document_stream_tests {
         ASSERT_SUCCESS( odparser.iterate_many(json.data(), json.length(), 50).get(odstream) );
         for (auto doc: odstream) {
             if(counter < 6) {
-                int64_t val;
+                int64_t val{};
                 ASSERT_SUCCESS(doc.at_pointer("/4").get(val));
                 ASSERT_EQUAL(val, 5);
             } else {

tests/ondemand/ondemand_misc_tests.cpp

@@ -67,7 +67,7 @@ namespace misc_tests {
     ASSERT_SUCCESS(val.get_object().get(obj));
     ondemand::array arr;
     ASSERT_SUCCESS(obj["a"].get_array().get(arr));
-    size_t count;
+    size_t count{};
     ASSERT_SUCCESS(arr.count_elements().get(count));
     ASSERT_EQUAL(3,count);
     TEST_SUCCEED();

tests/ondemand/ondemand_object_tests.cpp

@@ -6,6 +6,7 @@ using namespace simdjson;
 namespace object_tests {
   using namespace std;
   using simdjson::ondemand::json_type;
+
   bool issue1745() {
     TEST_START();
     auto json = R"({
@@ -225,6 +226,25 @@ namespace object_tests {
   }
 
 #if SIMDJSON_EXCEPTIONS
+
+  bool issue1965() {
+    TEST_START();
+    std::string str = "{\"query\":\"ah\"}";
+    std::unique_ptr<char[]> buffer(new char[str.size() + simdjson::SIMDJSON_PADDING]);
+    memcpy(buffer.get(), str.data(), str.size());
+    simdjson::padded_string_view view(buffer.get(), str.size(), str.size() + simdjson::SIMDJSON_PADDING);
+    simdjson::ondemand::parser parser;
+    simdjson::ondemand::document doc = parser.iterate(view);
+    simdjson::ondemand::object root = doc.get_object();
+    simdjson::ondemand::value query = root.find_field("query");
+    simdjson::ondemand::raw_json_string raw = query.get_raw_json_string();
+    std::unique_ptr<uint8_t[]> dst_buffer(new uint8_t[3 + simdjson::SIMDJSON_PADDING]);
+    uint8_t * dst = dst_buffer.get();
+    std::string_view fieldstring = parser.unescape(raw, dst);
+    std::cout << fieldstring << std::endl;
+    TEST_SUCCEED();
+  }
+
   bool issue1745_with_exceptions() {
     TEST_START();
     auto json = R"({
@@ -889,7 +909,7 @@ namespace object_tests {
     ondemand::document doc;
     ASSERT_SUCCESS(parser.iterate(json).get(doc));
     ondemand::object obj;
-    size_t count;
+    size_t count{};
     ASSERT_SUCCESS(doc.get_object().get(obj));
     ASSERT_SUCCESS(obj.count_fields().get(count));
     ASSERT_EQUAL(count, 0);
@@ -904,7 +924,7 @@ namespace object_tests {
     ASSERT_SUCCESS(parser.iterate(json).get(doc));
     ondemand::object obj;
     ASSERT_SUCCESS(doc.get_object().get(obj));
-    size_t count;
+    size_t count{};
     ASSERT_SUCCESS(obj.count_fields().get(count));
     ASSERT_EQUAL(count, 0);
     for (auto field : obj) {
@@ -923,7 +943,7 @@ namespace object_tests {
     ondemand::parser parser;
     ondemand::document doc;
     ASSERT_SUCCESS(parser.iterate(json).get(doc));
-    size_t count;
+    size_t count{};
     ASSERT_SUCCESS(doc.count_fields().get(count));
     ondemand::object obj;
     ASSERT_SUCCESS(doc.get_object().get(obj));
@@ -969,7 +989,7 @@ namespace object_tests {
     for (auto d : data) {
       simdjson::ondemand::object obj;
       ASSERT_SUCCESS(d.get_object().get(obj));
-      size_t count;
+      size_t count{};
       ASSERT_SUCCESS(obj.count_fields().get(count));
       ASSERT_EQUAL(count, 7);
     }
@@ -1005,7 +1025,7 @@ namespace object_tests {
     simdjson::ondemand::array data;
     ASSERT_SUCCESS(doc["result"]["data"].get_array().get(data));
     for (auto d : data) {
-      size_t count;
+      size_t count{};
       ASSERT_SUCCESS(d.count_fields().get(count));
       ASSERT_EQUAL(count, 7);
     }
@@ -1019,7 +1039,7 @@ namespace object_tests {
     ondemand::document doc;
     ASSERT_SUCCESS(parser.iterate(json).get(doc));
     ondemand::object obj;
-    size_t count;
+    size_t count{};
     ASSERT_SUCCESS(doc.get_object().get(obj));
     ASSERT_SUCCESS(obj.count_fields().get(count));
     ASSERT_EQUAL(count, 5);
@@ -1081,7 +1101,7 @@ namespace object_tests {
     TEST_START();
     auto empty = R"( {} )"_padded;
     SUBTEST("ondemand::empty_doc_object", test_ondemand_doc(empty, [&](auto doc_result) {
-        size_t count;
+        size_t count{};
         ASSERT_RESULT( doc_result.type(), json_type::object );
         ASSERT_SUCCESS( doc_result.count_fields().get(count) );
         ASSERT_EQUAL( count, 0 );
@@ -1089,7 +1109,7 @@ namespace object_tests {
     }));
     auto basic = R"( {"a":-1.234, "b":false, "c":null, "d":[1000.1,-2000.2,3000.3], "e":{"a":true, "b":false}} )"_padded;
     SUBTEST("ondemand::basic_doc_object", test_ondemand_doc(basic, [&](auto doc_result) {
-        size_t count;
+        size_t count{};
         ASSERT_RESULT( doc_result.type(), json_type::object );
         ASSERT_SUCCESS( doc_result.count_fields().get(count) );
         ASSERT_EQUAL( count, 5 );
@@ -1221,6 +1241,9 @@ namespace object_tests {
 
   bool run() {
     return
+#if SIMDJSON_EXCEPTIONS
+           issue1965() &&
+#endif
            issue1876a() &&
            issue1876() &&
            test_strager() &&

tests/ondemand/ondemand_readme_examples.cpp

@@ -320,7 +320,7 @@ bool json_object_count() {
   ondemand::parser parser;
   ondemand::document doc;
   ASSERT_SUCCESS(parser.iterate(json).get(doc));
-  size_t count;
+  size_t count{};
   ASSERT_SUCCESS(doc.count_fields().get(count));
   ASSERT_EQUAL(count,1);
   ondemand::object object;

tests/ondemand/ondemand_twitter_tests.cpp

@@ -79,7 +79,7 @@ namespace twitter_tests {
         auto media = tweet["entities"]["media"];
         if (!media.error()) {
           for (auto image : media) {
-            uint64_t id_val;
+            uint64_t id_val{};
             std::string_view id_string;
             ASSERT_SUCCESS( image["id"].get(id_val) );
             ASSERT_SUCCESS( image["id_str"].get(id_string) );
@@ -91,7 +91,8 @@ namespace twitter_tests {
               ASSERT_SUCCESS( size.unescaped_key().get(size_key) );
               std::cout << "Type of image size = " << size_key << std::endl;
 
-              uint64_t width, height;
+              uint64_t width{};
+              uint64_t height{};
               ASSERT_SUCCESS( size.value()["w"].get(width) );
               ASSERT_SUCCESS( size.value()["h"].get(height) );
               image_sizes.insert(make_pair(width, height));