Version 1.0.2.

* Fix for issue 1742.

* Some additional tests.

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 1.0.1
+    VERSION 1.0.2
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "1.0.1"
+PROJECT_NUMBER         = "1.0.2"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

include/simdjson/generic/ondemand/json_iterator-inl.h

@@ -53,17 +53,6 @@ inline void json_iterator::rewind() noexcept {
 SIMDJSON_PUSH_DISABLE_WARNINGS
 SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING
 simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child(depth_t parent_depth) noexcept {
-  /***
-   * WARNING:
-   * Inside an object, a string value is a depth of +1 compared to the object. Yet a key
-   * is at the same depth as the object.
-   * But json_iterator cannot easily tell whether we are pointing at a key or a string value.
-   * Instead, it assumes that if you are pointing at a string, then it is a value, not a key.
-   * To be clear...
-   * the following code assumes that we are *not* pointing at a key. If we are then a bug
-   * will follow. Unfortunately, it is not possible for the json_iterator its to make this
-   * check.
-   */
   if (depth() <= parent_depth) { return SUCCESS; }
   switch (*return_current_and_advance()) {
     // TODO consider whether matching braces is a requirement: if non-matching braces indicates
@@ -91,19 +80,18 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child
       if (at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "Missing [ or { at start"); }
 #endif // SIMDJSON_CHECK_EOF
       break;
-    /*case '"':
+    case '"':
       if(*peek() == ':') {
-        // we are at a key!!! This is
-        // only possible if someone searched
-        // for a key in an object and the key
-        // was not found but our code then
-        // decided the consume the separating
-        // comma before returning.
+        // We are at a key!!!
+        // This might happen if you just started an object and you skip it immediately.
+        // Performance note: it would be nice to get rid of this check as it is somewhat
+        // expensive.
+        // https://github.com/simdjson/simdjson/issues/1742
         logger::log_value(*this, "key");
-        advance(); // eat up the ':'
+        return_current_and_advance(); // eat up the ':'
         break; // important!!!
       }
-      simdjson_fallthrough;*/
+      simdjson_fallthrough;
     // Anything else must be a scalar value
     default:
       // For the first scalar, we will have incremented depth already, so we decrement it here.

include/simdjson/generic/ondemand/value-inl.h

@@ -114,6 +114,13 @@ simdjson_really_inline simdjson_result<size_t> value::count_elements() & noexcep
   iter.move_at_start();
   return answer;
 }
+simdjson_really_inline simdjson_result<size_t> value::count_fields() & noexcept {
+  simdjson_result<size_t> answer;
+  auto a = get_object();
+  answer = a.count_fields();
+  iter.move_at_start();
+  return answer;
+}
 simdjson_really_inline simdjson_result<value> value::at(size_t index) noexcept {
   auto a = get_array();
   return a.at(index);
@@ -169,6 +176,10 @@ simdjson_really_inline std::string_view value::raw_json_token() noexcept {
   return std::string_view(reinterpret_cast<const char*>(iter.peek_start()), iter.peek_start_length());
 }
 
+simdjson_really_inline simdjson_result<const char *> value::current_location() noexcept {
+  return iter.json_iter().current_location();
+}
+
 simdjson_really_inline simdjson_result<value> value::at_pointer(std::string_view json_pointer) noexcept {
   json_type t;
   SIMDJSON_TRY(type().get(t));
@@ -207,6 +218,10 @@ simdjson_really_inline simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEMEN
   if (error()) { return error(); }
   return first.count_elements();
 }
+simdjson_really_inline simdjson_result<size_t> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::count_fields() & noexcept {
+  if (error()) { return error(); }
+  return first.count_fields();
+}
 simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::at(size_t index) noexcept {
   if (error()) { return error(); }
   return first.at(index);
@@ -379,6 +394,11 @@ simdjson_really_inline simdjson_result<std::string_view> simdjson_result<SIMDJSO
   return first.raw_json_token();
 }
 
+simdjson_really_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::current_location() noexcept {
+  if (error()) { return error(); }
+  return first.current_location();
+}
+
 simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::at_pointer(std::string_view json_pointer) noexcept {
   if (error()) { return error(); }
   return first.at_pointer(json_pointer);

include/simdjson/generic/ondemand/value.h

@@ -244,6 +244,21 @@ public:
    * safe to continue.
    */
   simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
+  /**
+   * This method scans the object and counts the number of key-value pairs.
+   * The count_fields method should always be called before you have begun
+   * iterating through the object: it is expected that you are pointing at
+   * the beginning of the object.
+   * The runtime complexity is linear in the size of the object. After
+   * calling this function, if successful, the object is 'rewinded' at its
+   * beginning as if it had never been accessed. If the JSON is malformed (e.g.,
+   * there is a missing comma), then an error is returned and it is no longer
+   * safe to continue.
+   *
+   * To check that an object is empty, it is more performant to use
+   * the is_empty() method on the object instance.
+   */
+  simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
   /**
    * Get the value at the given index in the array. This function has linear-time complexity.
    * This function should only be called once as the array iterator is not reset between each call.
@@ -425,6 +440,11 @@ public:
    */
   simdjson_really_inline std::string_view raw_json_token() noexcept;
 
+  /**
+   * Returns the current location in the document if in bounds.
+   */
+  simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
+
   /**
    * Get the value associated with the given JSON pointer.  We use the RFC 6901
    * https://tools.ietf.org/html/rfc6901 standard.
@@ -552,6 +572,7 @@ public:
   simdjson_really_inline operator bool() noexcept(false);
 #endif
   simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
+  simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at(size_t index) noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> begin() & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> end() & noexcept;
@@ -624,6 +645,9 @@ public:
   /** @copydoc simdjson_really_inline std::string_view value::raw_json_token() const noexcept */
   simdjson_really_inline simdjson_result<std::string_view> raw_json_token() noexcept;
 
+  /** @copydoc simdjson_really_inline simdjson_result<const char *> current_location() noexcept */
+  simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
+
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at_pointer(std::string_view json_pointer) noexcept;
 };
 

include/simdjson/simdjson_version.h

@@ -4,7 +4,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION 1.0.1
+#define SIMDJSON_VERSION 1.0.2
 
 namespace simdjson {
 enum {
@@ -19,7 +19,7 @@ enum {
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 1
+  SIMDJSON_VERSION_REVISION = 2
 };
 } // namespace simdjson
 

singleheader/simdjson.cpp

@@ -1,4 +1,4 @@
-/* auto-generated on 2021-10-20 12:15:35 -0400. Do not edit! */
+/* auto-generated on 2021-10-27 19:25:23 -0400. Do not edit! */
 /* begin file src/simdjson.cpp */
 #include "simdjson.h"
 

singleheader/simdjson.h

@@ -1,4 +1,4 @@
-/* auto-generated on 2021-10-20 12:15:35 -0400. Do not edit! */
+/* auto-generated on 2021-10-27 19:25:23 -0400. Do not edit! */
 /* begin file include/simdjson.h */
 #ifndef SIMDJSON_H
 #define SIMDJSON_H
@@ -2189,7 +2189,7 @@ SIMDJSON_DISABLE_UNDESIRED_WARNINGS
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION 1.0.1
+#define SIMDJSON_VERSION 1.0.2
 
 namespace simdjson {
 enum {
@@ -2204,7 +2204,7 @@ enum {
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 1
+  SIMDJSON_VERSION_REVISION = 2
 };
 } // namespace simdjson
 
@@ -23584,6 +23584,21 @@ public:
    * safe to continue.
    */
   simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
+  /**
+   * This method scans the object and counts the number of key-value pairs.
+   * The count_fields method should always be called before you have begun
+   * iterating through the object: it is expected that you are pointing at
+   * the beginning of the object.
+   * The runtime complexity is linear in the size of the object. After
+   * calling this function, if successful, the object is 'rewinded' at its
+   * beginning as if it had never been accessed. If the JSON is malformed (e.g.,
+   * there is a missing comma), then an error is returned and it is no longer
+   * safe to continue.
+   *
+   * To check that an object is empty, it is more performant to use
+   * the is_empty() method on the object instance.
+   */
+  simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
   /**
    * Get the value at the given index in the array. This function has linear-time complexity.
    * This function should only be called once as the array iterator is not reset between each call.
@@ -23765,6 +23780,11 @@ public:
    */
   simdjson_really_inline std::string_view raw_json_token() noexcept;
 
+  /**
+   * Returns the current location in the document if in bounds.
+   */
+  simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
+
   /**
    * Get the value associated with the given JSON pointer.  We use the RFC 6901
    * https://tools.ietf.org/html/rfc6901 standard.
@@ -23892,6 +23912,7 @@ public:
   simdjson_really_inline operator bool() noexcept(false);
 #endif
   simdjson_really_inline simdjson_result<size_t> count_elements() & noexcept;
+  simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value> at(size_t index) noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array_iterator> begin() & noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array_iterator> end() & noexcept;
@@ -23964,6 +23985,9 @@ public:
   /** @copydoc simdjson_really_inline std::string_view value::raw_json_token() const noexcept */
   simdjson_really_inline simdjson_result<std::string_view> raw_json_token() noexcept;
 
+  /** @copydoc simdjson_really_inline simdjson_result<const char *> current_location() noexcept */
+  simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
+
   simdjson_really_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value> at_pointer(std::string_view json_pointer) noexcept;
 };
 
@@ -25660,17 +25684,6 @@ inline void json_iterator::rewind() noexcept {
 SIMDJSON_PUSH_DISABLE_WARNINGS
 SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING
 simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child(depth_t parent_depth) noexcept {
-  /***
-   * WARNING:
-   * Inside an object, a string value is a depth of +1 compared to the object. Yet a key
-   * is at the same depth as the object.
-   * But json_iterator cannot easily tell whether we are pointing at a key or a string value.
-   * Instead, it assumes that if you are pointing at a string, then it is a value, not a key.
-   * To be clear...
-   * the following code assumes that we are *not* pointing at a key. If we are then a bug
-   * will follow. Unfortunately, it is not possible for the json_iterator its to make this
-   * check.
-   */
   if (depth() <= parent_depth) { return SUCCESS; }
   switch (*return_current_and_advance()) {
     // TODO consider whether matching braces is a requirement: if non-matching braces indicates
@@ -25698,19 +25711,18 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child
       if (at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "Missing [ or { at start"); }
 #endif // SIMDJSON_CHECK_EOF
       break;
-    /*case '"':
+    case '"':
       if(*peek() == ':') {
-        // we are at a key!!! This is
-        // only possible if someone searched
-        // for a key in an object and the key
-        // was not found but our code then
-        // decided the consume the separating
-        // comma before returning.
+        // We are at a key!!!
+        // This might happen if you just started an object and you skip it immediately.
+        // Performance note: it would be nice to get rid of this check as it is somewhat
+        // expensive.
+        // https://github.com/simdjson/simdjson/issues/1742
         logger::log_value(*this, "key");
-        advance(); // eat up the ':'
+        return_current_and_advance(); // eat up the ':'
         break; // important!!!
       }
-      simdjson_fallthrough;*/
+      simdjson_fallthrough;
     // Anything else must be a scalar value
     default:
       // For the first scalar, we will have incremented depth already, so we decrement it here.
@@ -28148,6 +28160,13 @@ simdjson_really_inline simdjson_result<size_t> value::count_elements() & noexcep
   iter.move_at_start();
   return answer;
 }
+simdjson_really_inline simdjson_result<size_t> value::count_fields() & noexcept {
+  simdjson_result<size_t> answer;
+  auto a = get_object();
+  answer = a.count_fields();
+  iter.move_at_start();
+  return answer;
+}
 simdjson_really_inline simdjson_result<value> value::at(size_t index) noexcept {
   auto a = get_array();
   return a.at(index);
@@ -28203,6 +28222,10 @@ simdjson_really_inline std::string_view value::raw_json_token() noexcept {
   return std::string_view(reinterpret_cast<const char*>(iter.peek_start()), iter.peek_start_length());
 }
 
+simdjson_really_inline simdjson_result<const char *> value::current_location() noexcept {
+  return iter.json_iter().current_location();
+}
+
 simdjson_really_inline simdjson_result<value> value::at_pointer(std::string_view json_pointer) noexcept {
   json_type t;
   SIMDJSON_TRY(type().get(t));
@@ -28241,6 +28264,10 @@ simdjson_really_inline simdjson_result<size_t> simdjson_result<SIMDJSON_BUILTIN_
   if (error()) { return error(); }
   return first.count_elements();
 }
+simdjson_really_inline simdjson_result<size_t> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value>::count_fields() & noexcept {
+  if (error()) { return error(); }
+  return first.count_fields();
+}
 simdjson_really_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value>::at(size_t index) noexcept {
   if (error()) { return error(); }
   return first.at(index);
@@ -28413,6 +28440,11 @@ simdjson_really_inline simdjson_result<std::string_view> simdjson_result<SIMDJSO
   return first.raw_json_token();
 }
 
+simdjson_really_inline simdjson_result<const char *> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value>::current_location() noexcept {
+  if (error()) { return error(); }
+  return first.current_location();
+}
+
 simdjson_really_inline simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value>::at_pointer(std::string_view json_pointer) noexcept {
   if (error()) { return error(); }
   return first.at_pointer(json_pointer);

tests/ondemand/ondemand_array_tests.cpp

@@ -423,6 +423,36 @@ namespace array_tests {
     ASSERT_TRUE(is_empty);
     return true;
   }
+
+  bool issue1742() {
+    TEST_START();
+    auto json = R"( {
+  "code": 0,
+  "method": "subscribe",
+  "result": {
+    "instrument_name": "DAI_USDC",
+    "subscription": "trade.DAI_USDC",
+    "channel": "trade",
+    "data": [
+      [1,2,3,4]
+    ]
+  }
+} )"_padded;
+    ondemand::parser parser;
+    ondemand::document doc;
+    ASSERT_SUCCESS(parser.iterate(json).get(doc));
+    simdjson::ondemand::array data;
+    ASSERT_SUCCESS(doc["result"]["data"].get_array().get(data));
+    for (auto d : data) {
+      simdjson::ondemand::array arr;
+      ASSERT_SUCCESS(d.get_array().get(arr));
+      size_t count;
+      ASSERT_SUCCESS(arr.count_elements().get(count));
+      ASSERT_EQUAL(count, 4);
+    }
+    TEST_SUCCEED();
+  }
+
   bool value_to_array(simdjson::ondemand::value val) {
     ondemand::json_type t;
     ASSERT_SUCCESS(val.type().get(t));
@@ -780,6 +810,7 @@ namespace array_tests {
 
   bool run() {
     return
+           issue1742() &&
            empty_rewind_convoluted() &&
            empty_rewind() &&
            iterate_empty_array_count() &&

tests/ondemand/ondemand_error_location_tests.cpp

@@ -237,6 +237,27 @@ namespace error_location_tests {
         ASSERT_EQUAL(ptr, "13.34.514 ");
         TEST_SUCCEED();
     }
+    bool current_location_in_value() {
+      TEST_START();
+      auto json = R"( {"a": {"b": "c"}} )"_padded;
+      ondemand::parser parser;
+      ondemand::document doc;
+      ASSERT_SUCCESS(parser.iterate(json).get(doc));
+      simdjson_result<ondemand::value> a = doc["a"];
+      ASSERT_SUCCESS(a);
+      const char * ptr;
+      ASSERT_SUCCESS(a.current_location().get(ptr));
+      ASSERT_EQUAL(ptr, R"({"b": "c"}} )");
+      simdjson_result<ondemand::value> b = a["b"];
+      ASSERT_SUCCESS(b);
+      ASSERT_SUCCESS(b.current_location().get(ptr));
+      ASSERT_EQUAL(ptr, R"("c"}} )");
+      double d;
+      ASSERT_ERROR(b.get_double().get(d), INCORRECT_TYPE);
+      ASSERT_SUCCESS(b.current_location().get(ptr));
+      ASSERT_EQUAL(ptr, R"("c"}} )");
+      TEST_SUCCEED();
+    }
 
     bool run() {
         return  array() &&
@@ -253,6 +274,7 @@ namespace error_location_tests {
                 object_with_no_such_field() &&
                 number_parsing_error() &&
                 number_parsing_root_error() &&
+                current_location_in_value() &&
                 true;
     }
 

tests/ondemand/ondemand_object_tests.cpp

@@ -808,6 +808,79 @@ namespace object_tests {
     TEST_SUCCEED();
   }
 
+  bool issue1742() {
+    TEST_START();
+    auto json = R"( {
+  "code": 0,
+  "method": "subscribe",
+  "result": {
+    "instrument_name": "DAI_USDC",
+    "subscription": "trade.DAI_USDC",
+    "channel": "trade",
+    "data": [
+      {
+        "dataTime": 1635315736764,
+        "d": 1928673075051012900,
+        "s": "SELL",
+        "p": 1.0004,
+        "q": 0.01,
+        "t": 1635315736763,
+        "i": "DAI_USDC"
+      }
+    ]
+  }
+} )"_padded;
+    ondemand::parser parser;
+    ondemand::document doc;
+    ASSERT_SUCCESS(parser.iterate(json).get(doc));
+    simdjson::ondemand::array data;
+    ASSERT_SUCCESS(doc["result"]["data"].get_array().get(data));
+    for (auto d : data) {
+      simdjson::ondemand::object obj;
+      ASSERT_SUCCESS(d.get_object().get(obj));
+      size_t count;
+      ASSERT_SUCCESS(obj.count_fields().get(count));
+      ASSERT_EQUAL(count, 7);
+    }
+    TEST_SUCCEED();
+  }
+
+
+  bool issue1742_value() {
+    TEST_START();
+    auto json = R"( {
+  "code": 0,
+  "method": "subscribe",
+  "result": {
+    "instrument_name": "DAI_USDC",
+    "subscription": "trade.DAI_USDC",
+    "channel": "trade",
+    "data": [
+      {
+        "dataTime": 1635315736764,
+        "d": 1928673075051012900,
+        "s": "SELL",
+        "p": 1.0004,
+        "q": 0.01,
+        "t": 1635315736763,
+        "i": "DAI_USDC"
+      }
+    ]
+  }
+} )"_padded;
+    ondemand::parser parser;
+    ondemand::document doc;
+    ASSERT_SUCCESS(parser.iterate(json).get(doc));
+    simdjson::ondemand::array data;
+    ASSERT_SUCCESS(doc["result"]["data"].get_array().get(data));
+    for (auto d : data) {
+      size_t count;
+      ASSERT_SUCCESS(d.count_fields().get(count));
+      ASSERT_EQUAL(count, 7);
+    }
+    TEST_SUCCEED();
+  }
+
   bool iterate_basic_object_count() {
     TEST_START();
     auto json = R"( {"a":-55, "b":3.23, "c":100000000000000000000, "d":true, "e":null} )"_padded;
@@ -915,6 +988,8 @@ namespace object_tests {
 
   bool run() {
     return
+           issue1742() &&
+           issue1742_value() &&
            issue1723() &&
            value_search_unescaped_key() &&
            missing_key_continue() &&