diff --git a/doc/basics.md b/doc/basics.md index 91b53bb1e..776245be4 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -232,6 +232,15 @@ transcode the UTF-8 strings produced by the simdjson library to other formats. S Using the Parsed JSON --------------------- +We recommend that you first compile and run your code in Debug mode (with `NDEBUG` +undefined). When you do so, the simdjson library runs additional sanity tests on +your code to help ensure that you are using the library in a safe manner. Once +your code has been tested, you can then run it in Release mode (with `NDEBUG` +defined) for best performance. Alternatively, you can set the macro +`SIMDJSON_DEVELOPMENT_CHECKS` to 1 prior to including the `simdjson.h` header +to enable these additional checks: just make sure you remove the definition once your +code has been tested. + Once you have a document (`simdjson::ondemand::document`), you can navigate it with idiomatic C++ iterators, operators and casts. Besides the documents instances and native types (`double`, `uint64_t`, `int64_t`, `bool`), we also access @@ -268,7 +277,9 @@ support for users who avoid exceptions. See [the simdjson error handling documen * **Field Access:** To get the value of the "foo" field in an object, use `object["foo"]`. This will scan through the object looking for the field with the matching string, doing a character-by-character comparison. For efficiency reason, you should avoid looking up the same field repeatedly: e.g., do - not do `object["foo"]` followed by `object["foo"]` with the same `object` instance. + not do `object["foo"]` followed by `object["foo"]` with the same `object` instance. Furthermore, you can only consume one field at a time, on the same object. Thus + if you have retrieved `content["bids"].get_array()` and you later call `content["asks"].get_array()`, then the first array should no longer be accessed: it would be unsafe to do so. + You can detect such mistakes by first compiling and running the code in Debug mode: an OUT_OF_ORDER_ITERATION error is generated. > NOTE: JSON allows you to escape characters in keys. E.g., the key `"date"` may be written as > `"\u0064\u0061\u0074\u0065"`. By default, simdjson does *not* unescape keys when matching by default. @@ -683,30 +694,23 @@ auto cars_json = R"( [ ] )"_padded; ondemand::parser parser; -ondemand::document cars; std::vector measured; -parser.iterate(cars_json).get(cars); +ondemand::document cars = parser.iterate(cars_json); std::vector content; for (int i = 0; i < 3; i++) { - ondemand::object obj; std::string json_pointer = "/" + std::to_string(i); // Each successive at_pointer call invalidates // previously parsed values, strings, objects and array. - cars.at_pointer(json_pointer).get(obj); + ondemand::object obj(cars.at_pointer(json_pointer).get_object()); // We materialize the object. - std::string_view make; - ASSERT_SUCCESS(obj["make"].get(make)); - std::string_view model; - ASSERT_SUCCESS(obj["model"].get(model)); - uint64_t year; - ASSERT_SUCCESS(obj["year"].get(year)); + std::string_view make = obj["make"]; + std::string_view model = obj["model"]; + uint64_t year(obj["year"]); // We materialize the array. - ondemand::array arr; - ASSERT_SUCCESS(obj["tire_pressure"].get(arr)); + ondemand::array arr(obj["tire_pressure"].get_array()); std::vector values; for(auto x : arr) { - double value_double; - ASSERT_SUCCESS(x.get(value_double)); + double value_double(x.get_double()); values.push_back(value_double); } content.emplace_back(make, model, year, std::move(values)); diff --git a/include/simdjson/generic/ondemand/document.h b/include/simdjson/generic/ondemand/document.h index 570aecef0..885b405c9 100644 --- a/include/simdjson/generic/ondemand/document.h +++ b/include/simdjson/generic/ondemand/document.h @@ -303,6 +303,14 @@ public: * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys. * e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`. * + * + * You must consume the fields on an object one at a time. A request for a new key + * invalidates previous field values: it makes them unsafe. E.g., the array + * given by content["bids"].get_array() should not be accessed after you have called + * content["asks"].get_array(). You can detect such mistakes by first compiling and running + * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an + * OUT_OF_ORDER_ITERATION error is generated. + * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ @@ -326,6 +334,13 @@ public: * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the * field wasn't there when they aren't). * + * You must consume the fields on an object one at a time. A request for a new key + * invalidates previous field values: it makes them unsafe. E.g., the array + * given by content["bids"].get_array() should not be accessed after you have called + * content["asks"].get_array(). You can detect such mistakes by first compiling and running + * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an + * OUT_OF_ORDER_ITERATION error is generated. + * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ diff --git a/include/simdjson/generic/ondemand/object.h b/include/simdjson/generic/ondemand/object.h index ecb40220b..9c01b9c1d 100644 --- a/include/simdjson/generic/ondemand/object.h +++ b/include/simdjson/generic/ondemand/object.h @@ -37,6 +37,13 @@ public: * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys. * e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`. * + * You must consume the fields on an object one at a time. A request for a new key + * invalidates previous field values: it makes them unsafe. E.g., the array + * given by content["bids"].get_array() should not be accessed after you have called + * content["asks"].get_array(). You can detect such mistakes by first compiling and running + * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an + * OUT_OF_ORDER_ITERATION error is generated. + * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ @@ -63,6 +70,13 @@ public: * If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful * that only one field is returned. * + * You must consume the fields on an object one at a time. A request for a new key + * invalidates previous field values: it makes them unsafe. E.g., the array + * given by content["bids"].get_array() should not be accessed after you have called + * content["asks"].get_array(). You can detect such mistakes by first compiling and running + * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an + * OUT_OF_ORDER_ITERATION error is generated. + * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ diff --git a/tests/ondemand/ondemand_json_pointer_tests.cpp b/tests/ondemand/ondemand_json_pointer_tests.cpp index f6d808cff..179479ded 100644 --- a/tests/ondemand/ondemand_json_pointer_tests.cpp +++ b/tests/ondemand/ondemand_json_pointer_tests.cpp @@ -346,8 +346,49 @@ namespace json_pointer_tests { TEST_SUCCEED(); } +#if SIMDJSON_EXCEPTIONS + bool json_pointer_invalidation_exceptions() { + TEST_START(); + auto cars_json = R"( [ + { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, + { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, + { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] } + ] )"_padded; + + ondemand::parser parser; + std::vector measured; + ondemand::document cars = parser.iterate(cars_json); + std::vector content; + for (int i = 0; i < 3; i++) { + std::string json_pointer = "/" + std::to_string(i); + // Each successive at_pointer call invalidates + // previously parsed values, strings, objects and array. + ondemand::object obj(cars.at_pointer(json_pointer).get_object()); + // We materialize the object. + std::string_view make = obj["make"]; + std::string_view model = obj["model"]; + uint64_t year(obj["year"]); + // We materialize the array. + ondemand::array arr(obj["tire_pressure"].get_array()); + std::vector values; + for(auto x : arr) { + double value_double(x.get_double()); + values.push_back(value_double); + } + content.emplace_back(make, model, year, std::move(values)); + } + std::string expected[] = {"Toyota", "Kia", "Toyota"}; + for (car_type c : content) { + std::cout << c.make << " " << c.model << " " << c.year << "\n"; + } + TEST_SUCCEED(); + } +#endif bool run() { return +#if SIMDJSON_EXCEPTIONS + json_pointer_invalidation_exceptions() && +#endif many_json_pointers_array() && many_json_pointers_object() && many_json_pointers_object_array() && diff --git a/tests/ondemand/ondemand_object_tests.cpp b/tests/ondemand/ondemand_object_tests.cpp index 6b03451b7..9256392e2 100644 --- a/tests/ondemand/ondemand_object_tests.cpp +++ b/tests/ondemand/ondemand_object_tests.cpp @@ -6,6 +6,61 @@ using namespace simdjson; namespace object_tests { using namespace std; using simdjson::ondemand::json_type; + + bool issue1723() { + TEST_START(); + simdjson::ondemand::parser parser; + simdjson::padded_string docdata = R"({ + "contents": + { + "bids":[ + {"offset":"1", "price":"1.1", "size":"4.1"} + ], + "asks":[ + {"offset":"1111", "price":"1.100", "size":"114.1"} + ] + + } + })"_padded; + simdjson::ondemand::document doc; + auto error = parser.iterate(docdata).get(doc); + if(error) { return false; } + ondemand::object content; + error = doc.get_object().find_field("contents").get(content); + if(error) { return false; } + ondemand::array bids; + error = content["bids"].get_array().get(bids); + if(error) { return false; } + for (auto aux : bids) { + uint64_t id; + error = aux["offset"].get_uint64_in_string().get(id); + if(error) { return false; } + double price; + error = aux["price"].get_double_in_string().get(price); + if(error) { return false; } + double size; + error = aux["size"].get_double_in_string().get(size); + if(error) { return false; } + std::cout << id << " " << price << " " << size << std::endl; + } + ondemand::array asks; + error = content["asks"].get_array().get(asks); + if(error) { return false; } + for (auto aux : asks) { + uint64_t id; + error = aux["offset"].get_uint64_in_string().get(id); + if(error) { return false; } + double price; + error = aux["price"].get_double_in_string().get(price); + if(error) { return false; } + double size; + error = aux["size"].get_double_in_string().get(size); + if(error) { return false; } + std::cout << id << " " << price << " " << size << std::endl; + } + return true; + } + // In this test, no non-trivial object in an array have a missing key bool no_missing_keys() { TEST_START(); @@ -123,6 +178,42 @@ namespace object_tests { } #if SIMDJSON_EXCEPTIONS + + bool issue1723_except() { + TEST_START(); + simdjson::ondemand::parser parser; + simdjson::padded_string docdata = R"({ + "contents": + { + "bids":[ + {"offset":"1", "price":"1.1", "size":"4.1"} + ], + "asks":[ + {"offset":"1111", "price":"1.100", "size":"114.1"} + ] + + } + })"_padded; + simdjson::ondemand::document doc = parser.iterate(docdata); + ondemand::object content = doc.get_object().find_field("contents"); + ondemand::array bids = content["bids"].get_array(); + std::cout << "bids:" << std::endl; + for (auto aux : bids) { + uint64_t id = aux["offset"].get_uint64_in_string(); + double price = aux["price"].get_double_in_string(); + double size = aux["size"].get_double_in_string(); + std::cout << id << " " << price << " " << size << std::endl; + } + ondemand::array asks = content["asks"].get_array(); + std::cout << "asks:" << std::endl; + for (auto aux : asks) { + uint64_t id = aux["offset"].get_uint64_in_string(); + double price = aux["price"].get_double_in_string(); + double size = aux["size"].get_double_in_string(); + std::cout << id << " " << price << " " << size << std::endl; + } + return true; + } // used in issue_1521 // difficult to use as a lambda because it is recursive. void broken_descend(ondemand::object node) { @@ -824,12 +915,14 @@ namespace object_tests { bool run() { return + issue1723() && value_search_unescaped_key() && missing_key_continue() && no_missing_keys() && missing_keys() && missing_keys_for_empty_top_level_object() && #if SIMDJSON_EXCEPTIONS + issue1723_except() && fixed_broken_issue_1521() && issue_1521() && broken_issue_1521() &&