version bump

1. Allows processing inclomplete, damaged, corrupted json to some extent.
2. Pariity with the Presto Java functionality.
3. Protected with SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON define.
4. Does not interfere with the normal path (can co-exist).
5. Tested in production forkflow.

.github/workflows/ubuntu22-gcc13.yml

@@ -1,4 +1,4 @@
-name: Ubuntu 22.04 CI (GCC 13)
+name: Ubuntu 24.04 CI (GCC 13)
 
 on: [push, pull_request]
 
@@ -7,7 +7,7 @@ jobs:
     if: >-
       ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - uses: actions/cache@v4
@@ -20,4 +20,4 @@ jobs:
           cd build &&
           CXX=g++-13 cmake -DSIMDJSON_DEVELOPER_MODE=ON ..  &&
           cmake --build .   &&
-          ctest --output-on-failure -LE explicitonly -j
+          ctest --output-on-failure -LE explicitonly -j

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 3.9.2
+    VERSION 3.9.3
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "3.9.2"
+PROJECT_NUMBER         = "3.9.3"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

doc/basics.md

@@ -2622,4 +2622,4 @@ Further reading
 --------
 
 
-- John Keiser, Daniel Lemire, [On-Demand JSON: A Better Way to Parse Documents?](http://arxiv.org/abs/2312.17149), Software: Practice and Experience (to appear)
+- John Keiser, Daniel Lemire, [On-Demand JSON: A Better Way to Parse Documents?](http://arxiv.org/abs/2312.17149), Software: Practice and Experience 54 (6), 2024

include/simdjson/dom/document_stream-inl.h

@@ -224,10 +224,10 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc
   } else {
     size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index];
     size_t svlen = next_doc_index - current_index();
-    if(svlen > 1) {
+    while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0')) {
       svlen--;
     }
-    return std::string_view(reinterpret_cast<const char*>(stream->buf) + current_index(), svlen);
+    return std::string_view(start, svlen);
   }
 }
 

include/simdjson/dom/element-inl.h

@@ -375,6 +375,23 @@ inline simdjson_result<element> element::operator[](const char *key) const noexc
   return at_key(key);
 }
 
+inline bool is_pointer_well_formed(std::string_view json_pointer) noexcept {
+  if (simdjson_unlikely(json_pointer[0] != '/')) {
+    return false;
+  }
+  size_t escape = json_pointer.find('~');
+  if (escape == std::string_view::npos) {
+    return true;
+  }
+  if (escape == json_pointer.size() - 1) {
+    return false;
+  }
+  if (json_pointer[escape + 1] != '0' && json_pointer[escape + 1] != '1') {
+    return false;
+  }
+  return true;
+}
+
 inline simdjson_result<element> element::at_pointer(std::string_view json_pointer) const noexcept {
   SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914
   switch (tape.tape_ref_type()) {
@@ -383,7 +400,10 @@ inline simdjson_result<element> element::at_pointer(std::string_view json_pointe
     case internal::tape_type::START_ARRAY:
       return array(tape).at_pointer(json_pointer);
     default: {
-      if(!json_pointer.empty()) { // a non-empty string is invalid on an atom
+      if (!json_pointer.empty()) { // a non-empty string can be invalid, or accessing a primitive (issue 2154)
+        if (is_pointer_well_formed(json_pointer)) {
+          return NO_SUCH_FIELD;
+        }
         return INVALID_JSON_POINTER;
       }
       // an empty string means that we return the current node

include/simdjson/dom/serialization.h

@@ -57,15 +57,15 @@ public:
   simdjson_inline void one_char(char c);
 
   simdjson_inline void call_print_newline() {
-      this->print_newline();
+      static_cast<formatter*>(this)->print_newline();
   }
 
   simdjson_inline void call_print_indents(size_t depth) {
-      this->print_indents(depth);
+      static_cast<formatter*>(this)->print_indents(depth);
   }
 
   simdjson_inline void call_print_space() {
-      this->print_space();
+      static_cast<formatter*>(this)->print_space();
   }
 
 protected:

include/simdjson/error.h

@@ -39,7 +39,7 @@ enum error_code {
   INDEX_OUT_OF_BOUNDS,        ///< JSON array index too large
   NO_SUCH_FIELD,              ///< JSON field not found in object
   IO_ERROR,                   ///< Error reading a file
-  INVALID_JSON_POINTER,       ///< Invalid JSON pointer reference
+  INVALID_JSON_POINTER,       ///< Invalid JSON pointer syntax
   INVALID_URI_FRAGMENT,       ///< Invalid URI fragment
   UNEXPECTED_ERROR,           ///< indicative of a bug in simdjson
   PARSER_IN_USE,              ///< parser is already in use.

include/simdjson/generic/ondemand/document_stream-inl.h

@@ -348,10 +348,11 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc
           auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index];
           // normally the length would be next_index - current_index() - 1, except for the last document
           size_t svlen = next_index - current_index();
-          if(svlen > 1) {
+          const char *start = reinterpret_cast<const char*>(stream->buf) + current_index();
+          while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0')) {
             svlen--;
           }
-          return std::string_view(reinterpret_cast<const char*>(stream->buf) + current_index(), svlen);
+          return std::string_view(start, svlen);
         }
     }
     cur_struct_index++;

include/simdjson/generic/ondemand/json_iterator-inl.h

@@ -54,6 +54,23 @@ simdjson_inline json_iterator::json_iterator(const uint8_t *buf, ondemand::parse
 #endif
 }
 
+#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
+simdjson_inline json_iterator::json_iterator(const uint8_t *buf, ondemand::parser *_parser, bool streaming) noexcept
+    : token(buf, &_parser->implementation->structural_indexes[0]),
+      parser{_parser},
+      _string_buf_loc{parser->string_buf.get()},
+      _depth{1},
+      _root{parser->implementation->structural_indexes.get()},
+      _streaming{streaming}
+
+{
+  logger::log_headers();
+#if SIMDJSON_CHECK_EOF
+  assert_more_tokens();
+#endif
+}
+#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
+
 inline void json_iterator::rewind() noexcept {
   token.set_position( root_position() );
   logger::log_headers(); // We start again

include/simdjson/generic/ondemand/json_iterator.h

@@ -293,6 +293,9 @@ public:
   inline bool balanced() const noexcept;
 protected:
   simdjson_inline json_iterator(const uint8_t *buf, ondemand::parser *parser) noexcept;
+#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
+  simdjson_inline json_iterator(const uint8_t *buf, ondemand::parser *parser, bool streaming) noexcept;
+#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
   /// The last token before the end
   simdjson_inline token_position last_position() const noexcept;
   /// The token *at* the end. This points at gibberish and should only be used for comparison.

include/simdjson/generic/ondemand/parser-inl.h

@@ -58,6 +58,27 @@ simdjson_warn_unused simdjson_inline simdjson_result<document> parser::iterate(p
   return document::start({ reinterpret_cast<const uint8_t *>(json.data()), this });
 }
 
+#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
+simdjson_warn_unused simdjson_inline simdjson_result<document> parser::iterate_allow_incomplete_json(padded_string_view json) & noexcept {
+  if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; }
+
+  json.remove_utf8_bom();
+
+  // Allocate if needed
+  if (capacity() < json.length() || !string_buf) {
+    SIMDJSON_TRY( allocate(json.length(), max_depth()) );
+  }
+
+  // Run stage 1.
+  const simdjson::error_code err = implementation->stage1(reinterpret_cast<const uint8_t *>(json.data()), json.length(), stage1_mode::regular);
+  if (err) {
+    if (err != UNCLOSED_STRING)
+      return err;
+  }
+  return document::start({ reinterpret_cast<const uint8_t *>(json.data()), this, true });
+}
+#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
+
 simdjson_warn_unused simdjson_inline simdjson_result<document> parser::iterate(const char *json, size_t len, size_t allocated) & noexcept {
   return iterate(padded_string_view(json, len, allocated));
 }

include/simdjson/generic/ondemand/parser.h

@@ -98,6 +98,9 @@ public:
    *         - UNCLOSED_STRING if there is an unclosed string in the document.
    */
   simdjson_warn_unused simdjson_result<document> iterate(padded_string_view json) & noexcept;
+#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
+  simdjson_warn_unused simdjson_result<document> iterate_allow_incomplete_json(padded_string_view json) & noexcept;
+#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
   /** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
   simdjson_warn_unused simdjson_result<document> iterate(const char *json, size_t len, size_t capacity) & noexcept;
   /** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */

include/simdjson/generic/ondemand/value-inl.h

@@ -239,6 +239,26 @@ simdjson_inline int32_t value::current_depth() const noexcept{
   return iter.json_iter().depth();
 }
 
+inline bool is_pointer_well_formed(std::string_view json_pointer) noexcept {
+  if (simdjson_unlikely(json_pointer.empty())) { // can't be
+    return false;
+  }
+  if (simdjson_unlikely(json_pointer[0] != '/')) {
+    return false;
+  }
+  size_t escape = json_pointer.find('~');
+  if (escape == std::string_view::npos) {
+    return true;
+  }
+  if (escape == json_pointer.size() - 1) {
+    return false;
+  }
+  if (json_pointer[escape + 1] != '0' && json_pointer[escape + 1] != '1') {
+    return false;
+  }
+  return true;
+}
+
 simdjson_inline simdjson_result<value> value::at_pointer(std::string_view json_pointer) noexcept {
   json_type t;
   SIMDJSON_TRY(type().get(t));
@@ -249,6 +269,10 @@ simdjson_inline simdjson_result<value> value::at_pointer(std::string_view json_p
     case json_type::object:
       return (*this).get_object().at_pointer(json_pointer);
     default:
+      // a non-empty string can be invalid, or accessing a primitive (issue 2154)
+      if (is_pointer_well_formed(json_pointer)) {
+        return NO_SUCH_FIELD;
+      }
       return INVALID_JSON_POINTER;
   }
 }

include/simdjson/simdjson_version.h

@@ -4,7 +4,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION "3.9.2"
+#define SIMDJSON_VERSION "3.9.3"
 
 namespace simdjson {
 enum {
@@ -19,7 +19,7 @@ enum {
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 2
+  SIMDJSON_VERSION_REVISION = 3
 };
 } // namespace simdjson
 

singleheader/simdjson.cpp

@@ -1,4 +1,4 @@
-/* auto-generated on 2024-05-07 18:04:59 -0400. Do not edit! */
+/* auto-generated on 2024-05-30 10:52:38 -0400. Do not edit! */
 /* including simdjson.cpp:  */
 /* begin file simdjson.cpp */
 #define SIMDJSON_SRC_SIMDJSON_CPP
@@ -2359,7 +2359,7 @@ enum error_code {
   INDEX_OUT_OF_BOUNDS,        ///< JSON array index too large
   NO_SUCH_FIELD,              ///< JSON field not found in object
   IO_ERROR,                   ///< Error reading a file
-  INVALID_JSON_POINTER,       ///< Invalid JSON pointer reference
+  INVALID_JSON_POINTER,       ///< Invalid JSON pointer syntax
   INVALID_URI_FRAGMENT,       ///< Invalid URI fragment
   UNEXPECTED_ERROR,           ///< indicative of a bug in simdjson
   PARSER_IN_USE,              ///< parser is already in use.
@@ -6896,6 +6896,7 @@ static inline uint32_t detect_supported_architectures() {
 /* end file internal/isadetection.h */
 
 #include <initializer_list>
+#include <type_traits>
 
 namespace simdjson {
 

src/implementation.cpp

@@ -7,6 +7,7 @@
 #include <internal/isadetection.h>
 
 #include <initializer_list>
+#include <type_traits>
 
 namespace simdjson {
 

tests/dom/document_stream_tests.cpp

@@ -249,6 +249,27 @@ namespace document_stream_tests {
       TEST_SUCCEED();
   }
 
+  bool issue2181() {
+      TEST_START();
+      auto json = R"(1 2 34)"_padded;
+      simdjson::dom::parser parser;
+      simdjson::dom::document_stream stream;
+      ASSERT_SUCCESS(parser.parse_many(json).get(stream));
+      auto i = stream.begin();
+      size_t count{0};
+      std::vector<size_t> indexes = { 0, 2, 4 };
+      std::vector<std::string_view> expected = { "1", "2", "34" };
+
+      for(; i != stream.end(); ++i) {
+          auto doc = *i;
+          ASSERT_SUCCESS(doc);
+          ASSERT_TRUE(count < 3);
+          ASSERT_EQUAL(i.current_index(), indexes[count]);
+          ASSERT_EQUAL(i.source(), expected[count]);
+          count++;
+      }
+      TEST_SUCCEED();
+  }
   bool issue1310() {
     std::cout << "Running " << __func__ << std::endl;
     // hex  : 20 20 5B 20 33 2C 31 5D 20 22 22 22 22 22 22 22 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20
@@ -963,7 +984,8 @@ namespace document_stream_tests {
   }
 
   bool run() {
-    return issue2170() &&
+    return issue2181() &&
+           issue2170() &&
            skipbom() &&
            fuzzaccess() &&
            baby_fuzzer() &&

tests/dom/pointercheck.cpp

@@ -191,10 +191,40 @@ bool issue1142() {
   return true;
 }
 
+bool issue2154() { // mistakenly taking value as path should not raise INVALID_JSON_POINTER
+#if SIMDJSON_EXCEPTIONS
+  std::cout << "issue 2154" << std::endl;
+  auto example_json = R"__(
+      {
+        "obj": {
+          "s": "42",
+          "n": 42,
+          "f": 4.2
+        }
+      }
+    )__"_padded;
+  dom::parser parser;
+  dom::element example = parser.parse(example_json);
+  std::string_view sfield = example.at_pointer("/obj/s");
+  ASSERT_EQUAL(sfield, "42");
+  int64_t nfield = example.at_pointer("/obj/n");
+  ASSERT_EQUAL(nfield, 42);
+  ASSERT_ERROR(example.at_pointer("/obj/X/42").error(), NO_SUCH_FIELD);
+  ASSERT_ERROR(example.at_pointer("/obj/s/42").error(), NO_SUCH_FIELD);
+  ASSERT_ERROR(example.at_pointer("/obj/n/42").error(), NO_SUCH_FIELD);
+  ASSERT_ERROR(example.at_pointer("/obj/f/4.2").error(), NO_SUCH_FIELD);
+  ASSERT_ERROR(example.at_pointer("/obj/f/4~").error(), INVALID_JSON_POINTER);
+  ASSERT_ERROR(example.at_pointer("/obj/f/~").error(), INVALID_JSON_POINTER);
+  ASSERT_ERROR(example.at_pointer("/obj/f/~1").error(), NO_SUCH_FIELD);
+#endif
+  return true;
+}
+
 int main() {
   if (true
     && demo()
     && issue1142()
+    && issue2154()
 #ifdef SIMDJSON_ENABLE_DEPRECATED_API
     && legacy_support()
 #endif

tests/ondemand/ondemand_document_stream_tests.cpp

@@ -217,6 +217,27 @@ namespace document_stream_tests {
         TEST_SUCCEED();
     }
 
+    bool issue2181() {
+        TEST_START();
+        auto json = R"(1 2 34)"_padded;
+        ondemand::parser parser;
+        ondemand::document_stream stream;
+        ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
+        auto i = stream.begin();
+        size_t count{0};
+        std::vector<size_t> indexes = { 0, 2, 4 };
+        std::vector<std::string_view> expected = { "1", "2", "34" };
+
+        for(; i != stream.end(); ++i) {
+            ASSERT_SUCCESS(i.error());
+            ASSERT_TRUE(count < 3);
+            ASSERT_EQUAL(i.current_index(), indexes[count]);
+            ASSERT_EQUAL(i.source(), expected[count]);
+            count++;
+        }
+        TEST_SUCCEED();
+    }
+
     bool issue1977() {
         TEST_START();
         std::string json = R"( 1111 })";
@@ -902,6 +923,7 @@ namespace document_stream_tests {
 
     bool run() {
         return
+            issue2181() &&
             issue2170() &&
             issue2137() &&
             skipbom() &&

tests/ondemand/ondemand_json_pointer_tests.cpp

@@ -385,8 +385,38 @@ namespace json_pointer_tests {
         TEST_SUCCEED();
     }
 #endif
+
+    bool issue2154() { // mistakenly taking value as path should not raise INVALID_JSON_POINTER
+#if SIMDJSON_EXCEPTIONS
+      std::cout << "issue 2154" << std::endl;
+      auto example_json = R"__({
+            "obj": {
+                "s": "42",
+                "n": 42,
+                "f": 4.2
+            }
+        })__"_padded;
+        ondemand::parser parser;
+        ondemand::document doc;
+        ASSERT_SUCCESS(parser.iterate(example_json).get(doc));
+        std::string_view sfield = doc.at_pointer("/obj/s");
+        ASSERT_EQUAL(sfield, "42");
+        int64_t nfield = doc.at_pointer("/obj/n");
+        ASSERT_EQUAL(nfield, 42);
+        ASSERT_ERROR(doc.at_pointer("/obj/X/42").error(), NO_SUCH_FIELD);
+        ASSERT_ERROR(doc.at_pointer("/obj/s/42").error(), NO_SUCH_FIELD);
+        ASSERT_ERROR(doc.at_pointer("/obj/n/42").error(), NO_SUCH_FIELD);
+        ASSERT_ERROR(doc.at_pointer("/obj/f/4.2").error(), NO_SUCH_FIELD);
+        ASSERT_ERROR(doc.at_pointer("/obj/f/4~").error(), INVALID_JSON_POINTER);
+        ASSERT_ERROR(doc.at_pointer("/obj/f/~").error(), INVALID_JSON_POINTER);
+        ASSERT_ERROR(doc.at_pointer("/obj/f/~1").error(), NO_SUCH_FIELD);
+#endif
+      return true;
+    }
+
     bool run() {
         return
+                issue2154() &&
 #if SIMDJSON_EXCEPTIONS
                 json_pointer_invalidation_exceptions() &&
 #endif