This PR adds an 'at_end()' method. (#1978)

* This PR adds an 'at_end()' method.

* Adding 1111 }

* Tweaking test.

benchmark/json2msgpack/simdjson_ondemand.h

@@ -100,6 +100,9 @@ simdjson2msgpack::to_msgpack(const simdjson::padded_string &json,
     recursive_processor_ref(val);
 #endif
   }
+  if (!doc.at_end()) {
+     throw "There are unexpectedly tokens after the end of the json in the json2msgpack sample data";
+  }
   return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
 }
 

doc/basics.md

@@ -23,6 +23,7 @@ An overview of what you need to know to use simdjson, with examples.
     - [Disabling Exceptions](#disabling-exceptions)
     - [Exceptions](#exceptions)
     - [Current location in document](#current-location-in-document)
+    - [Checking for trailing content](#checking-for-trailing-content)
   - [Rewinding](#rewinding)
   - [Direct Access to the Raw String](#direct-access-to-the-raw-string)
   - [Newline-Delimited JSON (ndjson) and JSON lines](#newline-delimited-json-ndjson-and-json-lines)
@@ -1152,18 +1153,6 @@ if (error) {
 }
 ```
 
-
-  auto broken_json = R"( {"double": 13.06, false, "integer": -343} )"_padded;
-  ondemand::parser parser;
-  ondemand::document doc;
-  ASSERT_SUCCESS(parser.iterate(broken_json).get(doc));
-  const char * ptr;
-  int64_t i;
-  ASSERT_ERROR(doc["integer"].get_int64().get(i), TAPE_ERROR);
-  ASSERT_SUCCESS(doc.current_location().get(ptr));
-  std::string expected = "false, \"integer\": -343} ";
-  ASSERT_EQUAL(std::string(ptr,expected.size()), expected);
-
 You may also use `current_location()` with exceptions as follows:
 
 ```c++
@@ -1236,6 +1225,31 @@ contains control characters that must be escaped and UNCLOSED_STRING if there
 is an unclosed string in the document. We do not provide location information for these
 errors.
 
+### Checking for trailing content
+
+The parser validates all parsed content, but your code may exhaust the content while
+not having processed the entire document. Thus, as a final optional step, you may
+call `at_end()` on the document instance. If it returns `false`, then you may
+conclude that you have trailing content and that your document is not valid JSON.
+You may then use `doc.current_location()` to obtain a pointer to the start of the trailing
+content.
+
+```C++
+  auto json = R"([1, 2] foo ])"_padded;
+  ondemand::parser parser;
+  ondemand::document doc = parser.iterate(json);
+  ondemand::array array = doc.get_array();
+  for (uint64_t values : array) {
+    std::cout << values << std::endl;
+    }
+  if(!doc.at_end()) {
+    std::cerr << "trailing content at byte index " << doc.current_location() - json.data() << std::endl;
+  }
+```
+
+The `at_end()` method is equivalent to `doc.current_location().error() == simdjson::SUCCESS` but
+more convenient.
+
 Rewinding
 ----------
 

include/simdjson/generic/ondemand/document-inl.h

@@ -20,7 +20,7 @@ inline std::string document::to_debug_string() noexcept {
   return iter.to_string();
 }
 
-inline simdjson_result<const char *> document::current_location() noexcept {
+inline simdjson_result<const char *> document::current_location() const noexcept {
   return iter.current_location();
 }
 
@@ -28,6 +28,11 @@ inline int32_t document::current_depth() const noexcept {
   return iter.depth();
 }
 
+inline bool document::at_end() const noexcept {
+  return iter.at_end();
+}
+
+
 inline bool document::is_alive() noexcept {
   return iter.is_alive();
 }
@@ -50,12 +55,14 @@ simdjson_inline simdjson_result<value> document::get_value() noexcept {
   iter.assert_at_document_depth();
   switch (*iter.peek()) {
     case '[': {
+      // The following lines check that the document ends with ].
       auto value_iterator = get_root_value_iterator();
       auto error = value_iterator.check_root_array();
       if(error) { return error; }
       return value(get_root_value_iterator());
     }
     case '{': {
+      // The following lines would check that the document ends with }.
       auto value_iterator = get_root_value_iterator();
       auto error = value_iterator.check_root_object();
       if(error) { return error; }
@@ -505,6 +512,12 @@ simdjson_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IMPLEMENT
   return first.current_location();
 }
 
+simdjson_inline bool simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::at_end() const noexcept {
+  if (error()) { return error(); }
+  return first.at_end();
+}
+
+
 simdjson_inline int32_t simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::current_depth() const noexcept {
   if (error()) { return error(); }
   return first.current_depth();

include/simdjson/generic/ondemand/document.h

@@ -511,7 +511,14 @@ public:
   /**
    * Returns the current location in the document if in bounds.
    */
-  inline simdjson_result<const char *> current_location() noexcept;
+  inline simdjson_result<const char *> current_location() const noexcept;
+
+  /**
+   * Returns true if this document has been fully parsed.
+   * If you have consumed the whole document and at_end() returns
+   * false, then there may be trailing content.
+   */
+  inline bool at_end() const noexcept;
 
   /**
    * Returns the current depth in the document if in bounds.
@@ -720,6 +727,7 @@ public:
   simdjson_inline simdjson_result<bool> is_scalar() noexcept;
   simdjson_inline simdjson_result<const char *> current_location() noexcept;
   simdjson_inline int32_t current_depth() const noexcept;
+  simdjson_inline bool at_end() const noexcept;
   simdjson_inline bool is_negative() noexcept;
   simdjson_inline simdjson_result<bool> is_integer() noexcept;
   simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> get_number_type() noexcept;

include/simdjson/generic/ondemand/json_iterator-inl.h

@@ -208,7 +208,7 @@ inline std::string json_iterator::to_string() const noexcept {
           + std::string(" ]");
 }
 
-inline simdjson_result<const char *> json_iterator::current_location() noexcept {
+inline simdjson_result<const char *> json_iterator::current_location() const noexcept {
   if (!is_alive()) {    // Unrecoverable error
     if (!at_root()) {
       return reinterpret_cast<const char *>(token.peek(-1));

include/simdjson/generic/ondemand/json_iterator.h

@@ -261,7 +261,7 @@ public:
   /**
    * Returns the current location in the document if in bounds.
    */
-  inline simdjson_result<const char *> current_location() noexcept;
+  inline simdjson_result<const char *> current_location() const noexcept;
 
   /**
    * Updates this json iterator so that it is back at the beginning of the document,

include/simdjson/generic/ondemand/value_iterator-inl.h

@@ -40,6 +40,11 @@ simdjson_warn_unused simdjson_inline error_code value_iterator::check_root_objec
   // Note that adding a check for 'streaming' is not expensive since we only have at most
   // one root element.
   if ( ! _json_iter->streaming() ) {
+    // The following lines do not fully protect against garbage content within the
+    // object: e.g., `{"a":2} foo }`. Users concerned with garbage content should
+    // call `at_end()` on the document instance at the end of the processing to
+    // ensure that the processing has finished at the end.
+    //
     if (*_json_iter->peek_last() != '}') {
       _json_iter->abandon();
       return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing } at end");
@@ -431,6 +436,11 @@ simdjson_warn_unused simdjson_inline error_code value_iterator::check_root_array
   // Note that adding a check for 'streaming' is not expensive since we only have at most
   // one root element.
   if ( ! _json_iter->streaming() ) {
+    // The following lines do not fully protect against garbage content within the
+    // array: e.g., `[1, 2] foo]`. Users concerned with garbage content should
+    // also call `at_end()` on the document instance at the end of the processing to
+    // ensure that the processing has finished at the end.
+    //
     if (*_json_iter->peek_last() != ']') {
       _json_iter->abandon();
       return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing ] at end");

src/internal/error_tables.cpp

@@ -4,37 +4,37 @@ namespace simdjson {
 namespace internal {
 
   SIMDJSON_DLLIMPORTEXPORT const error_code_info error_codes[] {
-    { SUCCESS, "No error" },
-    { CAPACITY, "This parser can't support a document that big" },
-    { MEMALLOC, "Error allocating memory, we're most likely out of memory" },
-    { TAPE_ERROR, "The JSON document has an improper structure: missing or superfluous commas, braces, missing keys, etc." },
-    { DEPTH_ERROR, "The JSON document was too deep (too many nested objects and arrays)" },
-    { STRING_ERROR, "Problem while parsing a string" },
-    { T_ATOM_ERROR, "Problem while parsing an atom starting with the letter 't'" },
-    { F_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'f'" },
-    { N_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'n'" },
-    { NUMBER_ERROR, "Problem while parsing a number" },
-    { UTF8_ERROR, "The input is not valid UTF-8" },
-    { UNINITIALIZED, "Uninitialized" },
-    { EMPTY, "Empty: no JSON found" },
-    { UNESCAPED_CHARS, "Within strings, some characters must be escaped, we found unescaped characters" },
-    { UNCLOSED_STRING, "A string is opened, but never closed." },
-    { UNSUPPORTED_ARCHITECTURE, "simdjson does not have an implementation supported by this CPU architecture (perhaps it's a non-SIMD CPU?)." },
-    { INCORRECT_TYPE, "The JSON element does not have the requested type." },
-    { NUMBER_OUT_OF_RANGE, "The JSON number is too large or too small to fit within the requested type." },
-    { INDEX_OUT_OF_BOUNDS, "Attempted to access an element of a JSON array that is beyond its length." },
-    { NO_SUCH_FIELD, "The JSON field referenced does not exist in this object." },
-    { IO_ERROR, "Error reading the file." },
-    { INVALID_JSON_POINTER, "Invalid JSON pointer syntax." },
-    { INVALID_URI_FRAGMENT, "Invalid URI fragment syntax." },
-    { UNEXPECTED_ERROR, "Unexpected error, consider reporting this problem as you may have found a bug in simdjson" },
-    { PARSER_IN_USE, "Cannot parse a new document while a document is still in use." },
-    { OUT_OF_ORDER_ITERATION, "Objects and arrays can only be iterated when they are first encountered." },
-    { INSUFFICIENT_PADDING, "simdjson requires the input JSON string to have at least SIMDJSON_PADDING extra bytes allocated, beyond the string's length. Consider using the simdjson::padded_string class if needed." },
-    { INCOMPLETE_ARRAY_OR_OBJECT, "JSON document ended early in the middle of an object or array." },
-    { SCALAR_DOCUMENT_AS_VALUE, "A JSON document made of a scalar (number, Boolean, null or string) is treated as a value. Use get_bool(), get_double(), etc. on the document instead. "},
-    { OUT_OF_BOUNDS, "Attempted to access location outside of document."},
-    { TRAILING_CONTENT, "Unexpected trailing content in the JSON input."}
+    { SUCCESS, "SUCCESS: No error" },
+    { CAPACITY, "CAPACITY: This parser can't support a document that big" },
+    { MEMALLOC, "MEMALLOC: Error allocating memory, we're most likely out of memory" },
+    { TAPE_ERROR, "TAPE_ERROR: The JSON document has an improper structure: missing or superfluous commas, braces, missing keys, etc." },
+    { DEPTH_ERROR, "DEPTH_ERROR: The JSON document was too deep (too many nested objects and arrays)" },
+    { STRING_ERROR, "STRING_ERROR: Problem while parsing a string" },
+    { T_ATOM_ERROR, "T_ATOM_ERROR: Problem while parsing an atom starting with the letter 't'" },
+    { F_ATOM_ERROR, "F_ATOM_ERROR: Problem while parsing an atom starting with the letter 'f'" },
+    { N_ATOM_ERROR, "N_ATOM_ERROR: Problem while parsing an atom starting with the letter 'n'" },
+    { NUMBER_ERROR, "NUMBER_ERROR: Problem while parsing a number" },
+    { UTF8_ERROR, "UTF8_ERROR: The input is not valid UTF-8" },
+    { UNINITIALIZED, "UNINITIALIZED: Uninitialized" },
+    { EMPTY, "EMPTY: no JSON found" },
+    { UNESCAPED_CHARS, "UNESCAPED_CHARS: Within strings, some characters must be escaped, we found unescaped characters" },
+    { UNCLOSED_STRING, "UNCLOSED_STRING: A string is opened, but never closed." },
+    { UNSUPPORTED_ARCHITECTURE, "UNSUPPORTED_ARCHITECTURE: simdjson does not have an implementation supported by this CPU architecture. Please report this error to the core team as it should never happen." },
+    { INCORRECT_TYPE, "INCORRECT_TYPE: The JSON element does not have the requested type." },
+    { NUMBER_OUT_OF_RANGE, "NUMBER_OUT_OF_RANGE: The JSON number is too large or too small to fit within the requested type." },
+    { INDEX_OUT_OF_BOUNDS, "INDEX_OUT_OF_BOUNDS: Attempted to access an element of a JSON array that is beyond its length." },
+    { NO_SUCH_FIELD, "NO_SUCH_FIELD: The JSON field referenced does not exist in this object." },
+    { IO_ERROR, "IO_ERROR: Error reading the file." },
+    { INVALID_JSON_POINTER, "INVALID_JSON_POINTER: Invalid JSON pointer syntax." },
+    { INVALID_URI_FRAGMENT, "INVALID_URI_FRAGMENT: Invalid URI fragment syntax." },
+    { UNEXPECTED_ERROR, "UNEXPECTED_ERROR: Unexpected error, consider reporting this problem as you may have found a bug in simdjson" },
+    { PARSER_IN_USE, "PARSER_IN_USE: Cannot parse a new document while a document is still in use." },
+    { OUT_OF_ORDER_ITERATION, "OUT_OF_ORDER_ITERATION: Objects and arrays can only be iterated when they are first encountered." },
+    { INSUFFICIENT_PADDING, "INSUFFICIENT_PADDING: simdjson requires the input JSON string to have at least SIMDJSON_PADDING extra bytes allocated, beyond the string's length. Consider using the simdjson::padded_string class if needed." },
+    { INCOMPLETE_ARRAY_OR_OBJECT, "INCOMPLETE_ARRAY_OR_OBJECT: JSON document ended early in the middle of an object or array." },
+    { SCALAR_DOCUMENT_AS_VALUE, "SCALAR_DOCUMENT_AS_VALUE: A JSON document made of a scalar (number, Boolean, null or string) is treated as a value. Use get_bool(), get_double(), etc. on the document instead. "},
+    { OUT_OF_BOUNDS, "OUT_OF_BOUNDS: Attempt to access location outside of document."},
+    { TRAILING_CONTENT, "TRAILING_CONTENT: Unexpected trailing content in the JSON input."}
   }; // error_messages[]
 
 } // namespace internal

tests/ondemand/ondemand_array_tests.cpp

@@ -6,6 +6,20 @@ using namespace simdjson;
 namespace array_tests {
   using namespace std;
   using simdjson::ondemand::json_type;
+  bool issue1977() {
+    TEST_START();
+    auto json = R"([1, 2] foo ])"_padded;
+    ondemand::parser parser;
+    ondemand::document doc;
+    ASSERT_SUCCESS(parser.iterate(json).get(doc));
+    ondemand::array array;
+    ASSERT_SUCCESS(doc.get_array().get(array));
+    for (auto values : array) {
+      ASSERT_SUCCESS(values);
+    }
+    ASSERT_FALSE(doc.at_end());
+    TEST_SUCCEED();
+  }
   bool issue1588() {
     TEST_START();
     const auto json = R"({
@@ -830,6 +844,7 @@ namespace array_tests {
 
   bool run() {
     return
+           issue1977() &&
            issue1876() &&
            issue1742() &&
            empty_rewind_convoluted() &&

tests/ondemand/ondemand_document_stream_tests.cpp

@@ -193,6 +193,20 @@ namespace document_stream_tests {
         TEST_SUCCEED();
     }
 
+    bool issue1977() {
+        TEST_START();
+        std::string json = R"( 1111 })";
+        ondemand::parser odparser;
+        ondemand::document_stream odstream;
+        ASSERT_SUCCESS(odparser.iterate_many(json).get(odstream));
+
+        auto i = odstream.begin();
+        for (; i != odstream.end(); ++i) {
+            ASSERT_TRUE(false);
+        }
+        ASSERT_TRUE(i.current_index() == 0);
+        TEST_SUCCEED();
+    }
 
     bool issue1683() {
         TEST_START();
@@ -789,6 +803,7 @@ namespace document_stream_tests {
 
     bool run() {
         return
+            issue1977() &&
             string_with_trailing() &&
             uint64_with_trailing() &&
             int64_with_trailing() &&

tests/ondemand/ondemand_object_tests.cpp

@@ -7,6 +7,21 @@ namespace object_tests {
   using namespace std;
   using simdjson::ondemand::json_type;
 
+  bool issue1977() {
+    TEST_START();
+    auto json = R"({"1": 2} foo })"_padded;
+    ondemand::parser parser;
+    ondemand::document doc;
+    ASSERT_SUCCESS(parser.iterate(json).get(doc));
+    ondemand::object object;
+    ASSERT_SUCCESS(doc.get_object().get(object));
+    for (auto values : object) {
+      ASSERT_SUCCESS(values);
+    }
+    ASSERT_FALSE(doc.at_end());
+    TEST_SUCCEED();
+  }
+
   bool issue1745() {
     TEST_START();
     auto json = R"({
@@ -1263,7 +1278,7 @@ namespace object_tests {
   }
 
   bool run() {
-    return
+    return issue1977() &&
 #if SIMDJSON_EXCEPTIONS
            issue1965() &&
 #endif

tests/ondemand/ondemand_readme_examples.cpp

@@ -9,6 +9,19 @@ using error_code=simdjson::error_code;
 
 #if SIMDJSON_EXCEPTIONS
 
+  bool at_end() {
+    auto json = R"([1, 2] foo ])"_padded;
+    ondemand::parser parser;
+    ondemand::document doc = parser.iterate(json);
+    ondemand::array array = doc.get_array();
+    for (uint64_t values : array) {
+      std::cout << values << std::endl;
+    }
+    if(!doc.at_end()) {
+      std::cerr << "trailing content at byte index " << doc.current_location() - json.data() << std::endl;
+    }
+    TEST_SUCCEED();
+  }
   bool number_tests() {
     ondemand::parser parser;
     padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999])"_padded;
@@ -978,6 +991,7 @@ bool current_location_tape_error() {
   TEST_SUCCEED();
 }
 
+
 bool current_location_user_error() {
   TEST_START();
   auto json = R"( [1,2,3] )"_padded;
@@ -1208,6 +1222,7 @@ bool example1958() {
 bool run() {
   return true
 #if SIMDJSON_EXCEPTIONS
+    && at_end()
     && example1956() && example1958()
 //    && basics_1() // Fails because twitter.json isn't in current directory. Compile test only.
     &&  basics_treewalk()