admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1351 from simdjson/jkeiser/unordered-lookup

Browse Source 
Make `object["field"]` order-insensitive in On Demand
This commit is contained in:
John Keiser
2020-12-24 13:33:09 -08:00
committed by GitHub
parent 93807bf230 98666e84ea
commit 158a3c53d8
51 changed files with 3355 additions and 2515 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.appveyor.yml
+7 -7
View File
@@ -13,24 +13,24 @@ environment:
matrix:
- job_name: VS2019
CMAKE_ARGS: -A %Platform%
CMAKE_ARGS: -A %Platform%
- job_name: VS2019ARM
CMAKE_ARGS: -A ARM64 -DCMAKE_CROSSCOMPILING=1 -D SIMDJSON_GOOGLE_BENCHMARKS=OFF # Does Google Benchmark builds under VS ARM?
- job_name: VS2017 (Static, No Threads)
image: Visual Studio 2017
CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_ENABLE_THREADS=OFF
CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_ENABLE_THREADS=OFF
CTEST_ARGS: -LE explicitonly
- job_name: VS2019 (Win32)
platform: Win32
CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway.
CTEST_ARGS: -E "checkperf|ondemand_basictests"
CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway.
CTEST_ARGS: -LE explicitonly
- job_name: VS2019 (Win32, No Exceptions)
platform: Win32
CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF
CTEST_ARGS: -E "checkperf|ondemand_basictests"
CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF
CTEST_ARGS: -LE explicitonly
- job_name: VS2015
image: Visual Studio 2015
CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_ENABLE_THREADS=OFF
CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_ENABLE_THREADS=OFF
CTEST_ARGS: -LE explicitonly
build_script:
.github/workflows/mingw-ci.yml
+2 -2
View File
@@ -61,5 +61,5 @@ jobs:
mkdir build32
cd build32
cmake -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
cmake --build . --target parse_many_test jsoncheck basictests ondemand_basictests numberparsingcheck stringparsingcheck errortests integer_tests pointercheck --verbose
ctest -R "(parse_many_test|jsoncheck|basictests|stringparsingcheck|numberparsingcheck|errortests|integer_tests|pointercheck)" --output-on-failure
cmake --build . --target acceptance_tests --verbose
ctest -L acceptance --output-on-failure
.github/workflows/mingw64-ci.yml
+4 -4
View File
@@ -61,11 +61,11 @@ jobs:
mkdir build64
cd build64
cmake -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
cmake --build . --target parse_many_test jsoncheck basictests ondemand_basictests numberparsingcheck stringparsingcheck errortests integer_tests pointercheck --verbose
ctest -R "(parse_many_test|jsoncheck|basictests|stringparsingcheck|numberparsingcheck|errortests|integer_tests|pointercheck)" --output-on-failure
cmake --build . --target acceptance_tests --verbose
ctest -L acceptance --output-on-failure
cd ..
mkdir build64debug
cd build64debug
cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
cmake --build . --target parse_many_test jsoncheck basictests ondemand_basictests numberparsingcheck stringparsingcheck errortests integer_tests pointercheck --verbose
ctest -R "(parse_many_test|jsoncheck|basictests|stringparsingcheck|numberparsingcheck|errortests|integer_tests|pointercheck)" --output-on-failure
cmake --build . --target acceptance_tests --verbose
ctest -L acceptance --output-on-failure
benchmark/CMakeLists.txt
+4 -1
View File
@@ -1,13 +1,15 @@
include_directories( . linux )
link_libraries(simdjson-windows-headers test-data)
# bench_sax links against the source
if (TARGET benchmark::benchmark)
add_executable(bench_sax bench_sax.cpp)
target_link_libraries(bench_sax PRIVATE simdjson-internal-flags simdjson-include-source benchmark::benchmark)
endif (TARGET benchmark::benchmark)
# Everything else links against simdjson proper
link_libraries(simdjson simdjson-flags)
add_executable(benchfeatures benchfeatures.cpp)
add_executable(get_corpus_benchmark get_corpus_benchmark.cpp)
add_executable(perfdiff perfdiff.cpp)
@@ -42,6 +44,7 @@ endif()
if (TARGET benchmark::benchmark)
link_libraries(benchmark::benchmark)
add_subdirectory(largerandom)
add_executable(bench_parse_call bench_parse_call.cpp)
add_executable(bench_dom_api bench_dom_api.cpp)
add_executable(bench_ondemand bench_ondemand.cpp)
benchmark/bench_ondemand.cpp
+1
View File
@@ -12,6 +12,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
#include "partial_tweets/dom.h"
#include "largerandom/ondemand.h"
#include "largerandom/ondemand_unordered.h"
// #include "largerandom/iter.h"
#include "largerandom/dom.h"
benchmark/distinctuserid/ondemand.h
+4 -4
View File
@@ -33,15 +33,15 @@ simdjson_really_inline bool OnDemand::Run(const padded_string &json) {
ids.clear();
// Walk the document, parsing as we go
auto doc = parser.iterate(json);
for (ondemand::object tweet : doc["statuses"]) {
for (ondemand::object tweet : doc.find_field("statuses")) {
// We believe that all statuses have a matching
// user, and we are willing to throw when they do not.
ids.push_back(tweet["user"]["id"]);
ids.push_back(tweet.find_field("user").find_field("id"));
// Not all tweets have a "retweeted_status", but when they do
// we want to go and find the user within.
auto retweet = tweet["retweeted_status"];
auto retweet = tweet.find_field("retweeted_status");
if(!retweet.error()) {
ids.push_back(retweet["user"]["id"]);
ids.push_back(retweet.find_field("user").find_field("id"));
}
}
remove_duplicates(ids);
benchmark/find_tweet/ondemand.h
+3 -3
View File
@@ -33,9 +33,9 @@ simdjson_really_inline bool OnDemand::Run(const padded_string &json) {
text = "";
// Walk the document, parsing as we go
auto doc = parser.iterate(json);
for (ondemand::object tweet : doc["statuses"]) {
if (uint64_t(tweet["id"]) == TWEET_ID) {
text = tweet["text"];
for (ondemand::object tweet : doc.find_field("statuses")) {
if (uint64_t(tweet.find_field("id")) == TWEET_ID) {
text = tweet.find_field("text");
return true;
}
}
benchmark/kostya/ondemand.h
+6 -6
View File
@@ -27,8 +27,8 @@ simdjson_really_inline bool OnDemand::Run(const padded_string &json) {
using std::endl;
auto doc = parser.iterate(json);
for (ondemand::object coord : doc["coordinates"]) {
container.emplace_back(my_point{coord["x"], coord["y"], coord["z"]});
for (ondemand::object coord : doc.find_field("coordinates")) {
container.emplace_back(my_point{coord.find_field("x"), coord.find_field("y"), coord.find_field("z")});
}
return true;
@@ -56,10 +56,10 @@ simdjson_really_inline bool OnDemand::Run(const padded_string &json) {
count = 0;
auto doc = parser.iterate(json);
for (ondemand::object coord : doc["coordinates"]) {
sum.x += double(coord["x"]);
sum.y += double(coord["y"]);
sum.z += double(coord["z"]);
for (ondemand::object coord : doc.find_field("coordinates")) {
sum.x += double(coord.find_field("x"));
sum.y += double(coord.find_field("y"));
sum.z += double(coord.find_field("z"));
count++;
}
benchmark/largerandom/CMakeLists.txt
+5
View File
@@ -0,0 +1,5 @@
if (TARGET benchmark::benchmark)
link_libraries(benchmark::benchmark)
add_executable(bench_ondemand_largerandom bench_ondemand_largerandom.cpp)
add_executable(bench_ondemand_unordered_largerandom bench_ondemand_unordered_largerandom.cpp)
endif()
benchmark/largerandom/bench_ondemand_largerandom.cpp
+14
View File
@@ -0,0 +1,14 @@
#include "simdjson.h"
#include <iostream>
#include <sstream>
#include <random>
#include <vector>
SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
#include <benchmark/benchmark.h>
SIMDJSON_POP_DISABLE_WARNINGS
#define BENCHMARK_NO_DOM
#include "largerandom/ondemand.h"
BENCHMARK_MAIN();
benchmark/largerandom/bench_ondemand_unordered_largerandom.cpp
+14
View File
@@ -0,0 +1,14 @@
#include "simdjson.h"
#include <iostream>
#include <sstream>
#include <random>
#include <vector>
SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
#include <benchmark/benchmark.h>
SIMDJSON_POP_DISABLE_WARNINGS
#define BENCHMARK_NO_DOM
#include "largerandom/ondemand_unordered.h"
BENCHMARK_MAIN();
benchmark/largerandom/dom.h
-32
View File
@@ -32,38 +32,6 @@ simdjson_really_inline bool Dom::Run(const padded_string &json) {
BENCHMARK_TEMPLATE(LargeRandom, Dom);
namespace sum {
class Dom {
public:
simdjson_really_inline bool Run(const padded_string &json);
simdjson_really_inline my_point &Result() { return sum; }
simdjson_really_inline size_t ItemCount() { return count; }
private:
dom::parser parser{};
my_point sum{};
size_t count{};
};
simdjson_really_inline bool Dom::Run(const padded_string &json) {
sum = { 0, 0, 0 };
count = 0;
for (auto coord : parser.parse(json)) {
sum.x += double(coord["x"]);
sum.y += double(coord["y"]);
sum.z += double(coord["z"]);
count++;
}
return true;
}
BENCHMARK_TEMPLATE(LargeRandomSum, Dom);
} // namespace sum
} // namespace largerandom
#endif // SIMDJSON_EXCEPTIONS
benchmark/largerandom/iter.h
-39
View File
@@ -48,45 +48,6 @@ simdjson_really_inline bool Iter::Run(const padded_string &json) {
BENCHMARK_TEMPLATE(LargeRandom, Iter);
namespace sum {
class Iter {
public:
simdjson_really_inline bool Run(const padded_string &json);
simdjson_really_inline my_point &Result() { return sum; }
simdjson_really_inline size_t ItemCount() { return count; }
private:
ondemand::parser parser{};
my_point sum{};
size_t count{};
};
simdjson_really_inline bool Iter::Run(const padded_string &json) {
sum = {0,0,0};
count = 0;
auto iter = parser.iterate_raw(json).value();
if (!iter.start_array()) { return false; }
do {
if (!iter.start_object() || iter.field_key().value() != "x" || iter.field_value()) { return false; }
sum.x += iter.consume_double();
if (!iter.has_next_field() || iter.field_key().value() != "y" || iter.field_value()) { return false; }
sum.y += iter.consume_double();
if (!iter.has_next_field() || iter.field_key().value() != "z" || iter.field_value()) { return false; }
sum.z += iter.consume_double();
if (*iter.advance() != '}') { return false; }
count++;
} while (iter.has_next_element());
return true;
}
BENCHMARK_TEMPLATE(LargeRandomSum, Iter);
} // namespace sum
} // namespace largerandom
#endif // SIMDJSON_EXCEPTIONS
benchmark/largerandom/largerandom.h
+6 -10
View File
@@ -8,9 +8,6 @@
namespace largerandom {
template<typename T> static void LargeRandom(benchmark::State &state);
namespace sum {
template<typename T> static void LargeRandomSum(benchmark::State &state);
}
using namespace simdjson;
@@ -59,22 +56,21 @@ simdjson_unused static std::ostream &operator<<(std::ostream &o, const my_point
//
#include <vector>
#include "event_counter.h"
#ifndef BENCHMARK_NO_DOM
#include "dom.h"
#endif
#include "json_benchmark.h"
namespace largerandom {
template<typename T> static void LargeRandom(benchmark::State &state) {
#ifdef BENCHMARK_NO_DOM
JsonBenchmark<T, T>(state, get_built_json_array());
#else
JsonBenchmark<T, Dom>(state, get_built_json_array());
#endif
}
namespace sum {
template<typename T> static void LargeRandomSum(benchmark::State &state) {
JsonBenchmark<T, Dom>(state, get_built_json_array());
}
}
} // namespace largerandom
#endif // SIMDJSON_EXCEPTIONS
benchmark/largerandom/ondemand.h
+1 -34
View File
@@ -25,7 +25,7 @@ simdjson_really_inline bool OnDemand::Run(const padded_string &json) {
auto doc = parser.iterate(json);
for (ondemand::object coord : doc) {
container.emplace_back(my_point{coord["x"], coord["y"], coord["z"]});
container.emplace_back(my_point{coord.find_field("x"), coord.find_field("y"), coord.find_field("z")});
}
return true;
@@ -33,39 +33,6 @@ simdjson_really_inline bool OnDemand::Run(const padded_string &json) {
BENCHMARK_TEMPLATE(LargeRandom, OnDemand);
namespace sum {
class OnDemand {
public:
simdjson_really_inline bool Run(const padded_string &json);
simdjson_really_inline my_point &Result() { return sum; }
simdjson_really_inline size_t ItemCount() { return count; }
private:
ondemand::parser parser{};
my_point sum{};
size_t count{};
};
simdjson_really_inline bool OnDemand::Run(const padded_string &json) {
sum = {0,0,0};
count = 0;
auto doc = parser.iterate(json);
for (ondemand::object coord : doc.get_array()) {
sum.x += double(coord["x"]);
sum.y += double(coord["y"]);
sum.z += double(coord["z"]);
count++;
}
return true;
}
BENCHMARK_TEMPLATE(LargeRandomSum, OnDemand);
} // namespace sum
} // namespace largerandom
#endif // SIMDJSON_EXCEPTIONS
benchmark/largerandom/ondemand_unordered.h
+38
View File
@@ -0,0 +1,38 @@
#pragma once
#if SIMDJSON_EXCEPTIONS
#include "largerandom.h"
namespace largerandom {
using namespace simdjson;
using namespace simdjson::builtin;
class OnDemandUnordered {
public:
simdjson_really_inline bool Run(const padded_string &json);
simdjson_really_inline const std::vector<my_point> &Result() { return container; }
simdjson_really_inline size_t ItemCount() { return container.size(); }
private:
ondemand::parser parser{};
std::vector<my_point> container{};
};
simdjson_really_inline bool OnDemandUnordered::Run(const padded_string &json) {
container.clear();
auto doc = parser.iterate(json);
for (ondemand::object coord : doc) {
container.emplace_back(my_point{coord["x"], coord["y"], coord["z"]});
}
return true;
}
BENCHMARK_TEMPLATE(LargeRandom, OnDemandUnordered);
} // namespace largerandom
#endif // SIMDJSON_EXCEPTIONS
benchmark/partial_tweets/ondemand.h
+9 -9
View File
@@ -32,7 +32,7 @@ private:
}
simdjson_really_inline twitter_user read_user(ondemand::object user) {
return { user["id"], user["screen_name"] };
return { user.find_field("id"), user.find_field("screen_name") };
}
static inline bool displayed_implementation = false;
@@ -43,15 +43,15 @@ simdjson_really_inline bool OnDemand::Run(const padded_string &json) {
// Walk the document, parsing the tweets as we go
auto doc = parser.iterate(json);
for (ondemand::object tweet : doc["statuses"]) {
for (ondemand::object tweet : doc.find_field("statuses")) {
tweets.emplace_back(partial_tweets::tweet{
tweet["created_at"],
tweet["id"],
tweet["text"],
nullable_int(tweet["in_reply_to_status_id"]),
read_user(tweet["user"]),
tweet["retweet_count"],
tweet["favorite_count"]
tweet.find_field("created_at"),
tweet.find_field("id"),
tweet.find_field("text"),
nullable_int(tweet.find_field("in_reply_to_status_id")),
read_user(tweet.find_field("user")),
tweet.find_field("retweet_count"),
tweet.find_field("favorite_count")
});
}
return true;
cmake/add_compile_only_test.cmake
+8
View File
@@ -0,0 +1,8 @@
function(add_compile_only_test TEST_NAME)
add_test(
NAME ${TEST_NAME}
COMMAND ${CMAKE_COMMAND} --build . --target ${TEST_NAME} --config $<CONFIGURATION>
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
)
set_target_properties(${TEST_NAME} PROPERTIES EXCLUDE_FROM_ALL TRUE EXCLUDE_FROM_DEFAULT_BUILD TRUE)
endfunction()
cmake/add_cpp_test.cmake
+18 -11
View File
@@ -3,12 +3,12 @@
# SOURCES defaults to testname.cpp if not specified.
function(add_cpp_test TEST_NAME)
# Parse arguments
cmake_parse_arguments(PARSE_ARGV 1 ARGS "COMPILE_ONLY;LIBRARY;WILL_FAIL" "" "SOURCES;LABELS")
cmake_parse_arguments(PARSE_ARGV 1 ARGS "COMPILE_ONLY;LIBRARY;WILL_FAIL" "" "SOURCES;LABELS;DEPENDENCY_OF")
if (NOT ARGS_SOURCES)
list(APPEND ARGS_SOURCES ${TEST_NAME}.cpp)
endif()
if (ARGS_COMPILE_ONLY)
list(APPEND ${ARGS_LABELS} compile)
list(APPEND ${ARGS_LABELS} compile_only)
endif()
# Add the compile target
@@ -28,22 +28,29 @@ function(add_cpp_test TEST_NAME)
set_target_properties(${TEST_NAME} PROPERTIES EXCLUDE_FROM_ALL TRUE EXCLUDE_FROM_DEFAULT_BUILD TRUE)
else()
add_test(${TEST_NAME} ${TEST_NAME})
# Add to <label>_tests make targets
foreach(label ${ARGS_LABELS})
list(APPEND ARGS_DEPENDENCY_OF ${label})
endforeach(label ${ARGS_LABELS})
endif()
# Add to test labels
if (ARGS_LABELS)
set_property(TEST ${TEST_NAME} APPEND PROPERTY LABELS ${ARGS_LABELS})
endif()
# Add as a dependency of given targets
foreach(dependency_of ${ARGS_DEPENDENCY_OF})
if (NOT TARGET ${dependency_of}_tests)
add_custom_target(${dependency_of}_tests)
add_dependencies(all_tests ${dependency_of}_tests)
endif(NOT TARGET ${dependency_of}_tests)
add_dependencies(${dependency_of}_tests ${TEST_NAME})
endforeach(dependency_of ${ARGS_DEPENDENCY_OF})
# If it will fail, mark the test as such
if (ARGS_WILL_FAIL)
set_property(TEST ${TEST_NAME} PROPERTY WILL_FAIL TRUE)
endif()
endfunction()
function(add_compile_only_test TEST_NAME)
add_test(
NAME ${TEST_NAME}
COMMAND ${CMAKE_COMMAND} --build . --target ${TEST_NAME} --config $<CONFIGURATION>
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
)
set_target_properties(${TEST_NAME} PROPERTIES EXCLUDE_FROM_ALL TRUE EXCLUDE_FROM_DEFAULT_BUILD TRUE)
endfunction()
doc/ondemand.md
+5 -15
View File
@@ -17,9 +17,9 @@ To achieve ease of use, we mimicked the *form* of a traditional DOM API: you can
arrays, look up fields in objects, and extract native values like `double`, `uint64_t`, `string` and `bool`.
To achieve performance, we introduced some key limitations that make the DOM API *streaming*:
array/object iteration cannot be restarted, and fields must be looked up in order, and string/number
values can only be parsed once. If these limitations are acceptable to you, the On Demand API could
help you write maintainable applications with a computation efficiency that is difficult to surpass.
array/object iteration cannot be restarted, and string/number values can only be parsed once. If
these limitations are acceptable to you, the On Demand API could help you write maintainable
applications with a computation efficiency that is difficult to surpass.
A code example illustrates our API from a programmer's point of view:
@@ -714,9 +714,9 @@ We expect that the On Demand approach has many of the performance benefits of th
### Limitations of the On Demand Approach
The On Demand approach has some limitations:
The On Demand approach has some limitations:
* Because it operates in streaming mode, you only have access to the current element in the JSON document. Furthermore, the document is traversed in order so the code is sensitive to the order of the JSON nodes in the same manner as an event-based approach (e.g., SAX).
* Because it operates in streaming mode, you only have access to the current element in the JSON document. Furthermore, the document is traversed in order so the code is sensitive to the order of the JSON nodes in the same manner as an event-based approach (e.g., SAX). (The one exception to this is field lookup, which is more *performant* when the order of lookups matches the order of fields in the document, but which will still work with out-of-order fields, with a performance hit.)
* The On Demand approach is less safe than DOM: we only validate the components of the JSON document that are used and it is possible to begin ingesting an invalid document only to find out later that the document is invalid. Are you fine ingesting a large JSON document that starts with well formed JSON but ends with invalid JSON content?
There are currently additional technical limitations which we expect to resolve in future releases of the simdjson library:
@@ -724,16 +724,6 @@ There are currently additional technical limitations which we expect to resolve
* The simdjson library offers runtime dispatching which allows you to compile one binary and have it run at full speed on different processors, taking advantage of the specific features of the processor. The On Demand API does not have runtime dispatch support at this time. To benefit from the On Demand API, you must compile your code for a specific processor. E.g., if your processor supports AVX2 instructions, you should compile your binary executable with AVX2 instruction support (by using your compiler's commands). If you are sufficiently technically proficient, you can implement runtime dispatching within your application, by compiling your On Demand code for different processors.
* There is an initial phase which scans the entire document quickly, irrespective of the size of the document. We plan to break this phase into distinct steps for large files in a future release as we have done with other components of our API (e.g., `parse_many`).
* The On Demand API does not support JSON Pointer. This capability is currently limited to our core API.
* You should be mindful that the though your software might write the keys in a consistent manner, the [JSON specification](https://www.rfc-editor.org/rfc/rfc8259.txt) states that "JSON parsing libraries have been observed to differ as to whether or not they make the ordering of object members visible". The On Demand API will help the programmer handle unexpected JSON dialects by throwing an exception when the unexpected occurs, but the programmer is responsible for handling such cases: e.g., by rejecting the JSON input that does not follow the expected JSON dialect. We intend to help users who wish to use the On Demand API but require support for order-insensitive semantics, but in our current implementation support for out-of-order keys (if needed) must be provided by the programmer. Currently, one might proceed in the following manner as a fallback measure if keys can appear in any order:
```C++
for (ondemand::object my_object : doc["mykey"]) {
for (auto field : my_object) {
if (field.key() == "key_value1") { process1(field.value()); }
else if (field.key() == "key_value2") { process2(field.value()); }
else if (field.key() == "key_value3") { process3(field.value()); }
}
}
```
### Applicability of the On Demand Approach
examples/quickstart/CMakeLists.txt
+3 -3
View File
@@ -51,15 +51,15 @@ IF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
add_quickstart_test(quickstart quickstart.cpp)
add_quickstart_test(quickstart11 quickstart.cpp c++11)
add_quickstart_test(quickstart14 quickstart.cpp c++14)
set_property( TEST quickstart quickstart11 APPEND PROPERTY LABELS acceptance compiletests )
set_property( TEST quickstart quickstart11 APPEND PROPERTY LABELS acceptance compiletests no_mingw)
endif()
add_quickstart_test(quickstart_noexceptions quickstart_noexceptions.cpp "" true)
add_quickstart_test(quickstart_noexceptions11 quickstart_noexceptions.cpp c++11 true)
set_property( TEST quickstart_noexceptions APPEND PROPERTY LABELS acceptance compile )
set_property( TEST quickstart_noexceptions APPEND PROPERTY LABELS acceptance compile no_mingw)
add_quickstart_test(quickstart2_noexceptions quickstart2_noexceptions.cpp "" true)
add_quickstart_test(quickstart2_noexceptions11 quickstart2_noexceptions.cpp c++11 true)
set_property( TEST quickstart2_noexceptions APPEND PROPERTY LABELS acceptance compile )
set_property( TEST quickstart2_noexceptions APPEND PROPERTY LABELS acceptance compile no_mingw)
endif()
include/simdjson/common_defs.h
+2
View File
@@ -100,6 +100,7 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
#endif
#define SIMDJSON_DISABLE_DEPRECATED_WARNING SIMDJSON_DISABLE_VS_WARNING(4996)
#define SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING
#define SIMDJSON_POP_DISABLE_WARNINGS __pragma(warning( pop ))
#else // SIMDJSON_REGULAR_VISUAL_STUDIO
@@ -139,6 +140,7 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
#define SIMDJSON_DISABLE_UNDESIRED_WARNINGS
#endif
#define SIMDJSON_DISABLE_DEPRECATED_WARNING SIMDJSON_DISABLE_GCC_WARNING(-Wdeprecated-declarations)
#define SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING SIMDJSON_DISABLE_GCC_WARNING(-Wstrict-overflow)
#define SIMDJSON_POP_DISABLE_WARNINGS _Pragma("GCC diagnostic pop")
include/simdjson/generic/ondemand/document-inl.h
+49 -27
View File
@@ -12,43 +12,46 @@ simdjson_really_inline document document::start(json_iterator &&iter) noexcept {
return document(std::forward<json_iterator>(iter));
}
simdjson_really_inline value document::as_value() noexcept {
return as_value_iterator();
simdjson_really_inline value_iterator document::resume_value_iterator() noexcept {
return value_iterator(&iter, 1, iter.root_checkpoint());
}
simdjson_really_inline value_iterator document::as_value_iterator() noexcept {
simdjson_really_inline value_iterator document::get_root_value_iterator() noexcept {
iter.assert_at_root();
return value_iterator(&iter, 1, iter.root_checkpoint());
return resume_value_iterator();
}
simdjson_really_inline value_iterator document::as_non_root_value_iterator() noexcept {
return value_iterator(&iter, 1, iter.root_checkpoint());
simdjson_really_inline value document::resume_value() noexcept {
return resume_value_iterator();
}
simdjson_really_inline value document::get_root_value() noexcept {
return get_root_value_iterator();
}
simdjson_really_inline simdjson_result<array> document::get_array() & noexcept {
return as_value().get_array();
return get_root_value().get_array();
}
simdjson_really_inline simdjson_result<object> document::get_object() & noexcept {
return as_value().get_object();
return get_root_value().get_object();
}
simdjson_really_inline simdjson_result<uint64_t> document::get_uint64() noexcept {
return as_value_iterator().require_root_uint64();
return get_root_value_iterator().require_root_uint64();
}
simdjson_really_inline simdjson_result<int64_t> document::get_int64() noexcept {
return as_value_iterator().require_root_int64();
return get_root_value_iterator().require_root_int64();
}
simdjson_really_inline simdjson_result<double> document::get_double() noexcept {
return as_value_iterator().require_root_double();
return get_root_value_iterator().require_root_double();
}
simdjson_really_inline simdjson_result<std::string_view> document::get_string() & noexcept {
return as_value().get_string();
return get_root_value().get_string();
}
simdjson_really_inline simdjson_result<raw_json_string> document::get_raw_json_string() & noexcept {
return as_value().get_raw_json_string();
return get_root_value().get_raw_json_string();
}
simdjson_really_inline simdjson_result<bool> document::get_bool() noexcept {
return as_value_iterator().require_root_bool();
return get_root_value_iterator().require_root_bool();
}
simdjson_really_inline bool document::is_null() noexcept {
return as_value_iterator().is_root_null();
return get_root_value_iterator().is_root_null();
}
template<> simdjson_really_inline simdjson_result<array> document::get() & noexcept { return get_array(); }
@@ -89,21 +92,24 @@ simdjson_really_inline simdjson_result<array_iterator> document::begin() & noexc
simdjson_really_inline simdjson_result<array_iterator> document::end() & noexcept {
return {};
}
simdjson_really_inline simdjson_result<value> document::find_field(std::string_view key) & noexcept {
return resume_value().find_field(key);
}
simdjson_really_inline simdjson_result<value> document::find_field(const char *key) & noexcept {
return resume_value().find_field(key);
}
simdjson_really_inline simdjson_result<value> document::find_field_unordered(std::string_view key) & noexcept {
return resume_value().find_field_unordered(key);
}
simdjson_really_inline simdjson_result<value> document::find_field_unordered(const char *key) & noexcept {
return resume_value().find_field_unordered(key);
}
simdjson_really_inline simdjson_result<value> document::operator[](std::string_view key) & noexcept {
if (iter.at_root()) {
return get_object()[key];
} else {
// If we're not at the root, this is not the first key we've grabbed
return object::resume(as_non_root_value_iterator())[key];
}
return resume_value()[key];
}
simdjson_really_inline simdjson_result<value> document::operator[](const char *key) & noexcept {
if (iter.at_root()) {
return get_object()[key];
} else {
// If we're not at the root, this is not the first key we've grabbed
return object::resume(as_non_root_value_iterator())[key];
}
return resume_value()[key];
}
} // namespace ondemand
@@ -136,6 +142,14 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::end() & noexcept {
return {};
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::find_field_unordered(std::string_view key) & noexcept {
if (error()) { return error(); }
return first.find_field_unordered(key);
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::find_field_unordered(const char *key) & noexcept {
if (error()) { return error(); }
return first.find_field_unordered(key);
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::operator[](std::string_view key) & noexcept {
if (error()) { return error(); }
return first[key];
@@ -144,6 +158,14 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>
if (error()) { return error(); }
return first[key];
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::find_field(std::string_view key) & noexcept {
if (error()) { return error(); }
return first.find_field(key);
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::find_field(const char *key) & noexcept {
if (error()) { return error(); }
return first.find_field(key);
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get_array() & noexcept {
if (error()) { return error(); }
return first.get_array();
include/simdjson/generic/ondemand/document.h
+50 -12
View File
@@ -206,30 +206,64 @@ public:
simdjson_really_inline simdjson_result<array_iterator> end() & noexcept;
/**
* Look up a field by name on an object.
* Look up a field by name on an object (order-sensitive).
*
* Important notes:
* The following code reads z, then y, then x, and thus will not retrieve x or y if fed the
* JSON `{ "x": 1, "y": 2, "z": 3 }`:
*
* * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys.
* e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`.
* * **Once Only:** You may only look up a single field on a document. To look up multiple fields,
* use `.get_object()` or cast to `object`.
* ```c++
* simdjson::builtin::ondemand::parser parser;
* auto obj = parser.parse(R"( { "x": 1, "y": 2, "z": 3 } )"_padded);
* double z = obj.find_field("z");
* double y = obj.find_field("y");
* double x = obj.find_field("x");
* ```
*
* **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys.
* e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`.
*
* @param key The key to look up.
* @returns The value of the field, NO_SUCH_FIELD if the field is not in the object, or
* INCORRECT_TYPE if the JSON value is not an array.
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
*/
simdjson_really_inline simdjson_result<value> find_field(std::string_view key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<value> find_field(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> find_field(const char *key) & noexcept;
/**
* Look up a field by name on an object, without regard to key order.
*
* **Performance Notes:** This is a bit less performant than find_field(), though its effect varies
* and often appears negligible. It starts out normally, starting out at the last field; but if
* the field is not found, it scans from the beginning of the object to see if it missed it. That
* missing case has a non-cache-friendly bump and lots of extra scanning, especially if the object
* in question is large. The fact that the extra code is there also bumps the executable size.
*
* It is the default, however, because it would be highly surprising (and hard to debug) if the
* default behavior failed to look up a field just because it was in the wrong order--and many
* APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order.
*
* Use find_field() if you are sure fields will be in order (or are willing to treat it as if the
* field wasn't there when they aren't).
*
* @param key The key to look up.
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
*/
simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> find_field_unordered(const char *key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> operator[](std::string_view key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<value> operator[](std::string_view key) & noexcept; */
/** @overload simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> operator[](const char *key) & noexcept;
protected:
simdjson_really_inline document(ondemand::json_iterator &&iter) noexcept;
simdjson_really_inline const uint8_t *text(uint32_t idx) const noexcept;
simdjson_really_inline value as_value() noexcept;
simdjson_really_inline value_iterator as_value_iterator() noexcept;
simdjson_really_inline value_iterator as_non_root_value_iterator() noexcept;
simdjson_really_inline value_iterator resume_value_iterator() noexcept;
simdjson_really_inline value_iterator get_root_value_iterator() noexcept;
simdjson_really_inline value resume_value() noexcept;
simdjson_really_inline value get_root_value() noexcept;
static simdjson_really_inline document start(ondemand::json_iterator &&iter) noexcept;
//
@@ -290,8 +324,12 @@ public:
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> begin() & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> end() & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(std::string_view key) & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(const char *key) & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](std::string_view key) & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](const char *key) & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(const char *key) & noexcept;
};
} // namespace simdjson
include/simdjson/generic/ondemand/json_iterator-inl.h
+17
View File
@@ -29,6 +29,11 @@ simdjson_really_inline json_iterator::json_iterator(ondemand::parser *_parser) n
logger::log_headers();
}
// GCC 7 warns when the first line of this function is inlined away into oblivion due to the caller
// relating depth and parent_depth, which is a desired effect. The warning does not show up if the
// skip_child() function is not marked inline).
SIMDJSON_PUSH_DISABLE_WARNINGS
SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING
simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child(depth_t parent_depth) noexcept {
if (depth() <= parent_depth) { return SUCCESS; }
@@ -90,6 +95,8 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child
return report_error(TAPE_ERROR, "not enough close braces");
}
SIMDJSON_POP_DISABLE_WARNINGS
simdjson_really_inline bool json_iterator::at_root() const noexcept {
return token.checkpoint() == root_checkpoint();
}
@@ -132,11 +139,13 @@ simdjson_really_inline uint32_t json_iterator::peek_length(int32_t delta) const
}
simdjson_really_inline void json_iterator::ascend_to(depth_t parent_depth) noexcept {
SIMDJSON_ASSUME(parent_depth >= 0 && parent_depth < INT32_MAX - 1);
SIMDJSON_ASSUME(_depth == parent_depth + 1);
_depth = parent_depth;
}
simdjson_really_inline void json_iterator::descend_to(depth_t child_depth) noexcept {
SIMDJSON_ASSUME(child_depth >= 1 && child_depth < INT32_MAX);
SIMDJSON_ASSUME(_depth == child_depth - 1);
_depth = child_depth;
}
@@ -156,6 +165,14 @@ simdjson_really_inline error_code json_iterator::report_error(error_code _error,
return error;
}
simdjson_really_inline const uint32_t *json_iterator::checkpoint() const noexcept {
return token.checkpoint();
}
simdjson_really_inline void json_iterator::restore_checkpoint(const uint32_t *target_checkpoint) noexcept {
token.restore_checkpoint(target_checkpoint);
}
simdjson_really_inline error_code json_iterator::optional_error(error_code _error, const char *message) noexcept {
SIMDJSON_ASSUME(_error == INCORRECT_TYPE || _error == NO_SUCH_FIELD);
logger::log_error(*this, message);
include/simdjson/generic/ondemand/json_iterator.h
+3
View File
@@ -163,6 +163,9 @@ public:
template<int N> simdjson_warn_unused simdjson_really_inline bool peek_to_buffer(uint8_t (&tmpbuf)[N]) noexcept;
template<int N> simdjson_warn_unused simdjson_really_inline bool advance_to_buffer(uint8_t (&tmpbuf)[N]) noexcept;
simdjson_really_inline const uint32_t *checkpoint() const noexcept;
simdjson_really_inline void restore_checkpoint(const uint32_t *target_checkpoint) noexcept;
protected:
simdjson_really_inline json_iterator(ondemand::parser *parser) noexcept;
include/simdjson/generic/ondemand/object-inl.h
+35 -43
View File
@@ -2,55 +2,31 @@ namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace ondemand {
//
// ### Live States
//
// While iterating or looking up values, depth >= iter.depth. at_start may vary. Error is
// always SUCCESS:
//
// - Start: This is the state when the object is first found and the iterator is just past the {.
// In this state, at_start == true.
// - Next: After we hand a scalar value to the user, or an array/object which they then fully
// iterate over, the iterator is at the , or } before the next value. In this state,
// depth == iter.depth, at_start == false, and error == SUCCESS.
// - Unfinished Business: When we hand an array/object to the user which they do not fully
// iterate over, we need to finish that iteration by skipping child values until we reach the
// Next state. In this state, depth > iter.depth, at_start == false, and error == SUCCESS.
//
// ## Error States
//
// In error states, we will yield exactly one more value before stopping. iter.depth == depth
// and at_start is always false. We decrement after yielding the error, moving to the Finished
// state.
//
// - Chained Error: When the object iterator is part of an error chain--for example, in
// `for (auto tweet : doc["tweets"])`, where the tweet field may be missing or not be an
// object--we yield that error in the loop, exactly once. In this state, error != SUCCESS and
// iter.depth == depth, and at_start == false. We decrement depth when we yield the error.
// - Missing Comma Error: When the iterator ++ method discovers there is no comma between fields,
// we flag that as an error and treat it exactly the same as a Chained Error. In this state,
// error == TAPE_ERROR, iter.depth == depth, and at_start == false.
//
// Errors that occur while reading a field to give to the user (such as when the key is not a
// string or the field is missing a colon) are yielded immediately. Depth is then decremented,
// moving to the Finished state without transitioning through an Error state at all.
//
// ## Terminal State
//
// The terminal state has iter.depth < depth. at_start is always false.
//
// - Finished: When we have reached a }, we are finished. We signal this by decrementing depth.
// In this state, iter.depth < depth, at_start == false, and error == SUCCESS.
//
simdjson_really_inline simdjson_result<value> object::find_field_unordered(const std::string_view key) & noexcept {
bool has_value;
SIMDJSON_TRY( iter.find_field_unordered_raw(key).get(has_value) );
if (!has_value) { return NO_SUCH_FIELD; }
return value(iter.child());
}
simdjson_really_inline simdjson_result<value> object::find_field_unordered(const std::string_view key) && noexcept {
bool has_value;
SIMDJSON_TRY( iter.find_field_unordered_raw(key).get(has_value) );
if (!has_value) { return NO_SUCH_FIELD; }
return value(iter.child());
}
simdjson_really_inline simdjson_result<value> object::operator[](const std::string_view key) & noexcept {
return find_field_unordered(key);
}
simdjson_really_inline simdjson_result<value> object::operator[](const std::string_view key) && noexcept {
return std::forward<object>(*this).find_field_unordered(key);
}
simdjson_really_inline simdjson_result<value> object::find_field(const std::string_view key) & noexcept {
bool has_value;
SIMDJSON_TRY( iter.find_field_raw(key).get(has_value) );
if (!has_value) { return NO_SUCH_FIELD; }
return value(iter.child());
}
simdjson_really_inline simdjson_result<value> object::operator[](const std::string_view key) && noexcept {
simdjson_really_inline simdjson_result<value> object::find_field(const std::string_view key) && noexcept {
bool has_value;
SIMDJSON_TRY( iter.find_field_raw(key).get(has_value) );
if (!has_value) { return NO_SUCH_FIELD; }
@@ -108,6 +84,14 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object
if (error()) { return error(); }
return first.end();
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object>::find_field_unordered(std::string_view key) & noexcept {
if (error()) { return error(); }
return first.find_field_unordered(key);
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object>::find_field_unordered(std::string_view key) && noexcept {
if (error()) { return error(); }
return std::forward<SIMDJSON_IMPLEMENTATION::ondemand::object>(first).find_field_unordered(key);
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object>::operator[](std::string_view key) & noexcept {
if (error()) { return error(); }
return first[key];
@@ -116,5 +100,13 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>
if (error()) { return error(); }
return std::forward<SIMDJSON_IMPLEMENTATION::ondemand::object>(first)[key];
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object>::find_field(std::string_view key) & noexcept {
if (error()) { return error(); }
return first.find_field(key);
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object>::find_field(std::string_view key) && noexcept {
if (error()) { return error(); }
return std::forward<SIMDJSON_IMPLEMENTATION::ondemand::object>(first).find_field(key);
}
} // namespace simdjson
include/simdjson/generic/ondemand/object.h
+44 -15
View File
@@ -20,29 +20,54 @@ public:
simdjson_really_inline object_iterator end() noexcept;
/**
* Look up a field by name on an object.
* Look up a field by name on an object (order-sensitive).
*
* Important notes:
* The following code reads z, then y, then x, and thus will not retrieve x or y if fed the
* JSON `{ "x": 1, "y": 2, "z": 3 }`:
*
* * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys.
* e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`.
* * **Order Sensitive:** Each field lookup will only move forward in the object. In particular,
* the following code reads z, then y, then x, and thus will not retrieve x or y if fed the
* JSON `{ "x": 1, "y": 2, "z": 3 }`:
* ```c++
* simdjson::builtin::ondemand::parser parser;
* auto obj = parser.parse(R"( { "x": 1, "y": 2, "z": 3 } )"_padded);
* double z = obj.find_field("z");
* double y = obj.find_field("y");
* double x = obj.find_field("x");
* ```
*
* ```c++
* simdjson::builtin::ondemand::parser parser;
* auto obj = parser.parse(R"( { "x": 1, "y": 2, "z": 3 } )"_padded);
* double z = obj["z"];
* double y = obj["y"];
* double x = obj["x"];
* ```
* **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys.
* e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`.
*
* @param key The key to look up.
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
*/
simdjson_really_inline simdjson_result<value> find_field(std::string_view key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<value> find_field(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> find_field(std::string_view key) && noexcept;
/**
* Look up a field by name on an object, without regard to key order.
*
* **Performance Notes:** This is a bit less performant than find_field(), though its effect varies
* and often appears negligible. It starts out normally, starting out at the last field; but if
* the field is not found, it scans from the beginning of the object to see if it missed it. That
* missing case has a non-cache-friendly bump and lots of extra scanning, especially if the object
* in question is large. The fact that the extra code is there also bumps the executable size.
*
* It is the default, however, because it would be highly surprising (and hard to debug) if the
* default behavior failed to look up a field just because it was in the wrong order--and many
* APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order.
*
* Use find_field() if you are sure fields will be in order (or are willing to treat it as if the
* field wasn't there when they aren't).
*
* @param key The key to look up.
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
*/
simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) && noexcept;
/** @overload simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> operator[](std::string_view key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<value> operator[](std::string_view key) & noexcept; */
/** @overload simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> operator[](std::string_view key) && noexcept;
protected:
@@ -76,6 +101,10 @@ public:
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object_iterator> begin() noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object_iterator> end() noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(std::string_view key) & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(std::string_view key) && noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) && noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](std::string_view key) & noexcept;
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](std::string_view key) && noexcept;
};
include/simdjson/generic/ondemand/token_iterator-inl.h
+4
View File
@@ -43,6 +43,10 @@ simdjson_really_inline const uint32_t *token_iterator::checkpoint() const noexce
return index;
}
simdjson_really_inline void token_iterator::restore_checkpoint(const uint32_t *target_checkpoint) noexcept {
index = target_checkpoint;
}
} // namespace ondemand
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson
include/simdjson/generic/ondemand/token_iterator.h
+5
View File
@@ -54,6 +54,11 @@ public:
*/
simdjson_really_inline const uint32_t *checkpoint() const noexcept;
/**
* Reset to a previously saved index.
*/
simdjson_really_inline void restore_checkpoint(const uint32_t *target_checkpoint) noexcept;
// NOTE: we don't support a full C++ iterator interface, because we expect people to make
// different calls to advance the iterator based on *their own* state.
include/simdjson/generic/ondemand/value-inl.h
+86 -4
View File
@@ -6,6 +6,13 @@ simdjson_really_inline value::value(const value_iterator &_iter) noexcept
: iter{_iter}
{
}
simdjson_really_inline value value::start(const value_iterator &iter) noexcept {
return iter;
}
simdjson_really_inline value value::resume(const value_iterator &iter) noexcept {
return iter;
}
simdjson_really_inline simdjson_result<array> value::get_array() && noexcept {
return array::start(iter);
}
@@ -18,6 +25,21 @@ simdjson_really_inline simdjson_result<object> value::get_object() && noexcept {
simdjson_really_inline simdjson_result<object> value::get_object() & noexcept {
return object::try_start(iter);
}
simdjson_really_inline simdjson_result<object> value::start_or_resume_object() & noexcept {
if (iter.at_start()) {
return get_object();
} else {
return object::resume(iter);
}
}
simdjson_really_inline simdjson_result<object> value::start_or_resume_object() && noexcept {
if (iter.at_start()) {
return get_object();
} else {
return object::resume(iter);
}
}
simdjson_really_inline simdjson_result<raw_json_string> value::get_raw_json_string() && noexcept {
return iter.require_raw_json_string();
}
@@ -145,17 +167,43 @@ simdjson_really_inline simdjson_result<array_iterator> value::end() & noexcept {
return {};
}
simdjson_really_inline simdjson_result<value> value::find_field(std::string_view key) & noexcept {
return start_or_resume_object().find_field(key);
}
simdjson_really_inline simdjson_result<value> value::find_field(std::string_view key) && noexcept {
return std::forward<value>(*this).start_or_resume_object().find_field(key);
}
simdjson_really_inline simdjson_result<value> value::find_field(const char *key) & noexcept {
return start_or_resume_object().find_field(key);
}
simdjson_really_inline simdjson_result<value> value::find_field(const char *key) && noexcept {
return std::forward<value>(*this).start_or_resume_object().find_field(key);
}
simdjson_really_inline simdjson_result<value> value::find_field_unordered(std::string_view key) & noexcept {
return start_or_resume_object().find_field_unordered(key);
}
simdjson_really_inline simdjson_result<value> value::find_field_unordered(std::string_view key) && noexcept {
return std::forward<value>(*this).start_or_resume_object().find_field_unordered(key);
}
simdjson_really_inline simdjson_result<value> value::find_field_unordered(const char *key) & noexcept {
return start_or_resume_object().find_field_unordered(key);
}
simdjson_really_inline simdjson_result<value> value::find_field_unordered(const char *key) && noexcept {
return std::forward<value>(*this).start_or_resume_object().find_field_unordered(key);
}
simdjson_really_inline simdjson_result<value> value::operator[](std::string_view key) & noexcept {
return get_object()[key];
return start_or_resume_object()[key];
}
simdjson_really_inline simdjson_result<value> value::operator[](std::string_view key) && noexcept {
return std::forward<value>(*this).get_object()[key];
return std::forward<value>(*this).start_or_resume_object()[key];
}
simdjson_really_inline simdjson_result<value> value::operator[](const char *key) & noexcept {
return get_object()[key];
return start_or_resume_object()[key];
}
simdjson_really_inline simdjson_result<value> value::operator[](const char *key) && noexcept {
return std::forward<value>(*this).get_object()[key];
return std::forward<value>(*this).start_or_resume_object()[key];
}
} // namespace ondemand
@@ -188,6 +236,40 @@ simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_
return {};
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::find_field(std::string_view key) & noexcept {
if (error()) { return error(); }
return first.find_field(key);
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::find_field(std::string_view key) && noexcept {
if (error()) { return error(); }
return std::forward<SIMDJSON_IMPLEMENTATION::ondemand::value>(first).find_field(key);
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::find_field(const char *key) & noexcept {
if (error()) { return error(); }
return first.find_field(key);
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::find_field(const char *key) && noexcept {
if (error()) { return error(); }
return std::forward<SIMDJSON_IMPLEMENTATION::ondemand::value>(first).find_field(key);
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::find_field_unordered(std::string_view key) & noexcept {
if (error()) { return error(); }
return first.find_field_unordered(key);
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::find_field_unordered(std::string_view key) && noexcept {
if (error()) { return error(); }
return std::forward<SIMDJSON_IMPLEMENTATION::ondemand::value>(first).find_field_unordered(key);
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::find_field_unordered(const char *key) & noexcept {
if (error()) { return error(); }
return first.find_field_unordered(key);
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::find_field_unordered(const char *key) && noexcept {
if (error()) { return error(); }
return std::forward<SIMDJSON_IMPLEMENTATION::ondemand::value>(first).find_field_unordered(key);
}
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::operator[](std::string_view key) & noexcept {
if (error()) { return error(); }
return first[key];
include/simdjson/generic/ondemand/value.h
+123 -24
View File
@@ -245,32 +245,71 @@ public:
simdjson_really_inline simdjson_result<array_iterator> end() & noexcept;
/**
* Look up a field by name on an object.
* Look up a field by name on an object (order-sensitive).
*
* Important notes:
* The following code reads z, then y, then x, and thus will not retrieve x or y if fed the
* JSON `{ "x": 1, "y": 2, "z": 3 }`:
*
* * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys.
* e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`.
* * **Once Only:** You may only look up a single field on a value. To look up multiple fields,
* you must cast to object or call `.get_object()`.
* ```c++
* simdjson::builtin::ondemand::parser parser;
* auto obj = parser.parse(R"( { "x": 1, "y": 2, "z": 3 } )"_padded);
* double z = obj.find_field("z");
* double y = obj.find_field("y");
* double x = obj.find_field("x");
* ```
*
* **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys.
* e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`.
*
* @param key The key to look up.
* @returns The value of the field, NO_SUCH_FIELD if the field is not in the object, or
* INCORRECT_TYPE if the JSON value is not an array.
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
*/
simdjson_really_inline simdjson_result<value> find_field(std::string_view key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<value> find_field(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> find_field(std::string_view key) && noexcept;
/** @overload simdjson_really_inline simdjson_result<value> find_field(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> find_field(const char *key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<value> find_field(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> find_field(const char *key) && noexcept;
/**
* Look up a field by name on an object, without regard to key order.
*
* **Performance Notes:** This is a bit less performant than find_field(), though its effect varies
* and often appears negligible. It starts out normally, starting out at the last field; but if
* the field is not found, it scans from the beginning of the object to see if it missed it. That
* missing case has a non-cache-friendly bump and lots of extra scanning, especially if the object
* in question is large. The fact that the extra code is there also bumps the executable size.
*
* It is the default, however, because it would be highly surprising (and hard to debug) if the
* default behavior failed to look up a field just because it was in the wrong order--and many
* APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order.
*
* Use find_field() if you are sure fields will be in order (or are willing to treat it as if the
* field wasn't there when they aren't).
*
* @param key The key to look up.
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
*/
simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) && noexcept;
/** @overload simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> find_field_unordered(const char *key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> find_field_unordered(const char *key) && noexcept;
/** @overload simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> operator[](std::string_view key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<value> operator[](std::string_view key) & noexcept; */
/** @overload simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> operator[](std::string_view key) && noexcept;
/** @overload simdjson_really_inline simdjson_result<value> operator[](std::string_view key) & noexcept; */
/** @overload simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> operator[](const char *key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<value> operator[](std::string_view key) & noexcept; */
/** @overload simdjson_really_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<value> operator[](const char *key) && noexcept;
protected:
/**
* Create a value.
*
* Use value::read() instead of this.
*/
simdjson_really_inline value(const value_iterator &iter) noexcept;
@@ -279,6 +318,25 @@ protected:
*/
simdjson_really_inline void skip() noexcept;
/**
* Start a value at the current position.
*
* (It should already be started; this is just a self-documentation method.)
*/
static simdjson_really_inline value start(const value_iterator &iter) noexcept;
/**
* Resume a value.
*/
static simdjson_really_inline value resume(const value_iterator &iter) noexcept;
/**
* Get the object, starting or resuming it as necessary
*/
simdjson_really_inline simdjson_result<object> start_or_resume_object() & noexcept;
/** @overload simdjson_really_inline simdjson_result<object> start_or_resume_object() & noexcept; */
simdjson_really_inline simdjson_result<object> start_or_resume_object() && noexcept;
// simdjson_really_inline void log_value(const char *type) const noexcept;
// simdjson_really_inline void log_error(const char *message) const noexcept;
@@ -362,25 +420,66 @@ public:
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> end() & noexcept;
/**
* Look up a field by name on an object.
* Look up a field by name on an object (order-sensitive).
*
* Important notes:
* The following code reads z, then y, then x, and thus will not retrieve x or y if fed the
* JSON `{ "x": 1, "y": 2, "z": 3 }`:
*
* * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys.
* e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`.
* * **Once Only:** You may only look up a single field on a value. To look up multiple fields,
* you must cast to object or call `.get_object()`.
* ```c++
* simdjson::builtin::ondemand::parser parser;
* auto obj = parser.parse(R"( { "x": 1, "y": 2, "z": 3 } )"_padded);
* double z = obj.find_field("z");
* double y = obj.find_field("y");
* double x = obj.find_field("x");
* ```
*
* **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys.
* e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`.
*
* @param key The key to look up.
* @returns The value of the field, NO_SUCH_FIELD if the field is not in the object, or
* INCORRECT_TYPE if the JSON value is not an array.
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
*/
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(std::string_view key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(std::string_view key) && noexcept;
/** @overload simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(const char *key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(const char *key) && noexcept;
/**
* Look up a field by name on an object, without regard to key order.
*
* **Performance Notes:** This is a bit less performant than find_field(), though its effect varies
* and often appears negligible. It starts out normally, starting out at the last field; but if
* the field is not found, it scans from the beginning of the object to see if it missed it. That
* missing case has a non-cache-friendly bump and lots of extra scanning, especially if the object
* in question is large. The fact that the extra code is there also bumps the executable size.
*
* It is the default, however, because it would be highly surprising (and hard to debug) if the
* default behavior failed to look up a field just because it was in the wrong order--and many
* APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order.
*
* Use find_field() if you are sure fields will be in order (or are willing to treat it as if the
* field wasn't there when they aren't).
*
* @param key The key to look up.
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
*/
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) && noexcept;
/** @overload simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(const char *key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(const char *key) && noexcept;
/** @overload simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](std::string_view key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<value> operator[](std::string_view key) & noexcept; */
/** @overload simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](std::string_view key) && noexcept;
/** @overload simdjson_really_inline simdjson_result<value> operator[](std::string_view key) & noexcept; */
/** @overload simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](const char *key) & noexcept;
/** @overload simdjson_really_inline simdjson_result<value> operator[](std::string_view key) & noexcept; */
/** @overload simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept; */
simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](const char *key) && noexcept;
};
include/simdjson/generic/ondemand/value_iterator-inl.h
+170 -8
View File
@@ -51,32 +51,64 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
}
}
/**
* Find the field with the given key. May be used in place of ++.
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::find_field_raw(const std::string_view key) noexcept {
if (!is_open()) { return false; }
// Unless this is the first field, we need to advance past the , and check for }
error_code error;
bool has_value;
//
// Initially, the object can be in one of a few different places:
//
// 1. The start of the object, at the first field:
//
// ```
// { "a": [ 1, 2 ], "b": [ 3, 4 ] }
// ^ (depth 2, index 1)
// ```
//
// 2. When a previous search did not yield a value or the object is empty:
//
// ```
// { "a": [ 1, 2 ], "b": [ 3, 4 ] }
// ^ (depth 0)
// { }
// ^ (depth 0, index 2)
// ```
//
if (!is_open()) { return false; }
if (at_first_field()) {
has_value = true;
// 3. When a previous search found a field or an iterator yielded a value:
//
// ```
// // When a field was not fully consumed (or not even touched at all)
// { "a": [ 1, 2 ], "b": [ 3, 4 ] }
// ^ (depth 2)
// // When a field was fully consumed
// { "a": [ 1, 2 ], "b": [ 3, 4 ] }
// ^ (depth 1)
// // When the last field was fully consumed
// { "a": [ 1, 2 ], "b": [ 3, 4 ] }
// ^ (depth 1)
// ```
//
} else {
if ((error = skip_child() )) { abandon(); return error; }
if ((error = has_next_field().get(has_value) )) { abandon(); return error; }
}
while (has_value) {
// Get the key
// Get the key and colon, stopping at the value.
raw_json_string actual_key;
if ((error = field_key().get(actual_key) )) { abandon(); return error; };
if ((error = field_value() )) { abandon(); return error; }
// Check if it matches
// If it matches, stop and return
if (actual_key == key) {
logger::log_event(*this, "match", key, -2);
return true;
}
// No match: skip the value and see if , or } is next
logger::log_event(*this, "no match", key, -2);
SIMDJSON_TRY( skip_child() ); // Skip the value entirely
if ((error = has_next_field().get(has_value) )) { abandon(); return error; }
@@ -86,6 +118,128 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
return false;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::find_field_unordered_raw(const std::string_view key) noexcept {
error_code error;
bool has_value;
//
// Initially, the object can be in one of a few different places:
//
// 1. The start of the object, at the first field:
//
// ```
// { "a": [ 1, 2 ], "b": [ 3, 4 ] }
// ^ (depth 2, index 1)
// ```
//
if (at_first_field()) {
// If we're at the beginning of the object, we definitely have a field
has_value = true;
// 2. When a previous search did not yield a value or the object is empty:
//
// ```
// { "a": [ 1, 2 ], "b": [ 3, 4 ] }
// ^ (depth 0)
// { }
// ^ (depth 0, index 2)
// ```
//
} else if (!is_open()) {
has_value = false;
// 3. When a previous search found a field or an iterator yielded a value:
//
// ```
// // When a field was not fully consumed (or not even touched at all)
// { "a": [ 1, 2 ], "b": [ 3, 4 ] }
// ^ (depth 2)
// // When a field was fully consumed
// { "a": [ 1, 2 ], "b": [ 3, 4 ] }
// ^ (depth 1)
// // When the last field was fully consumed
// { "a": [ 1, 2 ], "b": [ 3, 4 ] }
// ^ (depth 1)
// ```
//
} else {
// Finish the previous value and see if , or } is next
if ((error = skip_child() )) { abandon(); return error; }
if ((error = has_next_field().get(has_value) )) { abandon(); return error; }
}
// After initial processing, we will be in one of two states:
//
// ```
// // At the beginning of a field
// { "a": [ 1, 2 ], "b": [ 3, 4 ] }
// ^ (depth 1)
// { "a": [ 1, 2 ], "b": [ 3, 4 ] }
// ^ (depth 1)
// // At the end of the object
// { "a": [ 1, 2 ], "b": [ 3, 4 ] }
// ^ (depth 0)
// ```
//
// First, we scan from that point to the end.
// If we don't find a match, we loop back around, and scan from the beginning to that point.
const uint32_t *search_start = _json_iter->checkpoint();
// Next, we find a match starting from the current position.
while (has_value) {
SIMDJSON_ASSUME( _json_iter->_depth == _depth + 1 ); // We must be at the start of a field
// Get the key and colon, stopping at the value.
raw_json_string actual_key;
if ((error = field_key().get(actual_key) )) { abandon(); return error; };
if ((error = field_value() )) { abandon(); return error; }
// If it matches, stop and return
if (actual_key == key) {
logger::log_event(*this, "match", key, -2);
return true;
}
// No match: skip the value and see if , or } is next
logger::log_event(*this, "no match", key, -2);
SIMDJSON_TRY( skip_child() );
if ((error = has_next_field().get(has_value) )) { abandon(); return error; }
}
// If we reach the end without finding a match, search the rest of the fields starting at the
// beginning of the object.
// (We have already run through the object before, so we've already validated its structure. We
// don't check errors in this bit.)
_json_iter->restore_checkpoint(_start_index + 1);
_json_iter->descend_to(_depth);
has_value = started_object();
while (_json_iter->checkpoint() < search_start) {
SIMDJSON_ASSUME(has_value); // we should reach search_start before ever reaching the end of the object
SIMDJSON_ASSUME( _json_iter->_depth == _depth + 1 ); // We must be at the start of a field
// Get the key and colon, stopping at the value.
raw_json_string actual_key;
error = field_key().get(actual_key); SIMDJSON_ASSUME(!error);
error = field_value(); SIMDJSON_ASSUME(!error);
// If it matches, stop and return
if (actual_key == key) {
logger::log_event(*this, "match", key, -2);
return true;
}
// No match: skip the value and see if , or } is next
logger::log_event(*this, "no match", key, -2);
SIMDJSON_TRY( skip_child() );
error = has_next_field().get(has_value); SIMDJSON_ASSUME(!error);
}
// If the loop ended, we're out of fields to look at.
return false;
}
simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> value_iterator::field_key() noexcept {
assert_at_child();
@@ -389,6 +543,14 @@ simdjson_really_inline bool value_iterator::is_open() const noexcept {
return _json_iter->depth() >= depth();
}
simdjson_really_inline bool value_iterator::at_eof() const noexcept {
return _json_iter->at_eof();
}
simdjson_really_inline bool value_iterator::at_start() const noexcept {
return _json_iter->token.index == _start_index;
}
simdjson_really_inline bool value_iterator::at_first_field() const noexcept {
SIMDJSON_ASSUME( _json_iter->token.index > _start_index );
return _json_iter->token.index == _start_index + 1;
include/simdjson/generic/ondemand/value_iterator.h
+27 -1
View File
@@ -50,6 +50,11 @@ public:
*/
simdjson_really_inline bool at_eof() const noexcept;
/**
* Tell whether the iterator is at the start of the value
*/
simdjson_really_inline bool at_start() const noexcept;
/**
* Tell whether the value is open--if the value has not been used, or the array/object is still open.
*/
@@ -148,7 +153,8 @@ public:
simdjson_warn_unused simdjson_really_inline error_code find_field(const std::string_view key) noexcept;
/**
* Find the next field with the given key, *without* unescaping.
* Find the next field with the given key, *without* unescaping. This assumes object order: it
* will not find the field if it was already passed when looking for some *other* field.
*
* Assumes you have called next_field() or otherwise matched the previous value.
*
@@ -165,6 +171,26 @@ public:
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> find_field_raw(const std::string_view key) noexcept;
/**
* Find the field with the given key without regard to order, and *without* unescaping.
*
* This is an unordered object lookup: if the field is not found initially, it will cycle around and scan from the beginning.
*
* Assumes you have called next_field() or otherwise matched the previous value.
*
* This means the iterator must be sitting at the next key:
*
* ```
* { "a": 1, "b": 2 }
* ^
* ```
*
* Key is *raw JSON,* meaning it will be matched against the verbatim JSON without attempting to
* unescape it. This works well for typical ASCII and UTF-8 keys (almost all of them), but may
* fail to match some keys with escapes (\u, \n, etc.).
*/
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> find_field_unordered_raw(const std::string_view key) noexcept;
/** @} */
/**
singleheader/CMakeLists.txt
+2 -2
View File
@@ -1,5 +1,3 @@
include(${PROJECT_SOURCE_DIR}/cmake/add_cpp_test.cmake)
#
# Amalgamation
#
@@ -117,6 +115,8 @@ target_link_libraries(simdjson-singleheader-source INTERFACE simdjson-singlehead
#
#
include(${PROJECT_SOURCE_DIR}/cmake/add_compile_only_test.cmake)
#
# Test the generated simdjson.cpp/simdjson.h using the generated amalgamate_demo.cpp
#
src/CMakeLists.txt
+1 -1
View File
@@ -101,7 +101,7 @@ if(NOT MSVC)
# Please do not delete the following, our users want version numbers. See
# https://github.com/simdjson/simdjson/issues/1014
# https://github.com/simdjson/simdjson/issues/52
###########
###########
set_target_properties(simdjson PROPERTIES VERSION ${SIMDJSON_LIB_VERSION} SOVERSION ${SIMDJSON_LIB_SOVERSION})
##########
# End of the do-not-delete message.
tests/CMakeLists.txt
+22 -54
View File
@@ -1,45 +1,10 @@
# Helper so we don't have to repeat ourselves so much
# Usage: add_cpp_test(testname [COMPILE_ONLY] [SOURCES a.cpp b.cpp ...] [LABELS acceptance per_implementation ...])
# SOURCES defaults to testname.cpp if not specified.
function(add_cpp_test TEST_NAME)
# Parse arguments
cmake_parse_arguments(PARSE_ARGV 1 ARGS "COMPILE_ONLY;WILL_FAIL" "" "SOURCES;LABELS")
if (NOT ARGS_SOURCES)
list(APPEND ARGS_SOURCES ${TEST_NAME}.cpp)
endif()
if (COMPILE_ONLY)
list(APPEND ${ARGS_LABELS} compile)
endif()
# Add executable
add_executable(${TEST_NAME} ${ARGS_SOURCES})
# Add test
if (ARGS_COMPILE_ONLY)
add_test(
NAME ${TEST_NAME}
COMMAND ${CMAKE_COMMAND} --build . --target ${TEST_NAME} --config $<CONFIGURATION>
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
)
set_target_properties(${TEST_NAME} PROPERTIES EXCLUDE_FROM_ALL TRUE EXCLUDE_FROM_DEFAULT_BUILD TRUE)
else()
add_test(${TEST_NAME} ${TEST_NAME})
endif()
if (ARGS_LABELS)
set_property(TEST ${TEST_NAME} APPEND PROPERTY LABELS ${ARGS_LABELS})
endif()
if (ARGS_WILL_FAIL)
set_property(TEST ${TEST_NAME} PROPERTY WILL_FAIL TRUE)
endif()
endfunction()
# Most tests need test data, and many need windows headers.
link_libraries(simdjson-internal-flags test-data simdjson-windows-headers)
include(${PROJECT_SOURCE_DIR}/cmake/add_cpp_test.cmake)
# add_cpp_tests will add dependencies to this
add_custom_target(all_tests)
add_subdirectory(ondemand)
#
@@ -54,20 +19,21 @@ target_compile_definitions(stringparsingcheck PRIVATE NOMINMAX)
# All remaining tests link with simdjson proper
link_libraries(simdjson)
add_cpp_test(random_string_number_tests LABELS acceptance per_implementation)
add_cpp_test(basictests LABELS acceptance per_implementation)
add_cpp_test(minify_tests LABELS acceptance per_implementation)
add_cpp_test(document_stream_tests LABELS acceptance per_implementation)
add_cpp_test(document_tests LABELS acceptance per_implementation)
add_cpp_test(errortests LABELS acceptance per_implementation)
add_cpp_test(integer_tests LABELS acceptance per_implementation)
add_cpp_test(jsoncheck LABELS acceptance per_implementation)
add_cpp_test(minefieldcheck LABELS acceptance per_implementation)
add_cpp_test(parse_many_test LABELS acceptance per_implementation)
add_cpp_test(pointercheck LABELS acceptance per_implementation) # https://tools.ietf.org/html/rfc6901
add_cpp_test(extracting_values_example LABELS acceptance per_implementation)
add_cpp_test(unicode_tests LABELS acceptance per_implementation)
add_cpp_test(padded_string_tests)
add_cpp_test(random_string_number_tests LABELS dom acceptance per_implementation)
add_cpp_test(basictests LABELS dom acceptance per_implementation)
add_cpp_test(document_stream_tests LABELS dom acceptance per_implementation)
add_cpp_test(document_tests LABELS dom acceptance per_implementation)
add_cpp_test(errortests LABELS dom acceptance per_implementation)
add_cpp_test(extracting_values_example LABELS dom acceptance per_implementation)
add_cpp_test(integer_tests LABELS dom acceptance per_implementation)
add_cpp_test(jsoncheck LABELS dom acceptance per_implementation)
add_cpp_test(minefieldcheck LABELS dom acceptance per_implementation)
add_cpp_test(parse_many_test LABELS dom acceptance per_implementation)
add_cpp_test(pointercheck LABELS dom acceptance per_implementation) # https://tools.ietf.org/html/rfc6901
add_cpp_test(unicode_tests LABELS dom acceptance per_implementation)
add_cpp_test(minify_tests LABELS other acceptance per_implementation)
add_cpp_test(padded_string_tests LABELS other acceptance )
find_program(BASH bash)
@@ -94,11 +60,13 @@ if (BASH AND (NOT WIN32) AND SIMDJSON_BASH AND (TARGET json2json)) # The scripts
if ((SIMDJSON_COMPETITION) AND (!SIMDJSON_SANITIZE))
# It looks like RapidJSON does not pass the sanitizer under some conditions (Clang 10)
add_executable(allparserscheckfile allparserscheckfile.cpp)
add_dependencies(competition_tests allparserscheckfile)
add_dependencies(per_implementation_tests allparserscheckfile)
target_link_libraries(allparserscheckfile PRIVATE competition-all)
add_test(issue150 ${BASH} ${CMAKE_CURRENT_SOURCE_DIR}/issue150.sh)
set_property(TEST issue150 APPEND PROPERTY DEPENDS allparserscheckfile)
set_property(TEST issue150 APPEND PROPERTY LABELS per_implementation)
set_property(TEST issue150 APPEND PROPERTY LABELS per_implementation competition)
endif()
#
@@ -107,7 +75,7 @@ if (BASH AND (NOT WIN32) AND SIMDJSON_BASH AND (TARGET json2json)) # The scripts
# This tests validates that the implementation is what we think it is if we get passed
# SIMDJSON_FORCE_IMPLEMENTATION, so we know we're testing what we think we're testing
add_cpp_test(checkimplementation LABELS per_implementation)
add_cpp_test(checkimplementation LABELS other per_implementation)
add_test(NAME json2json COMMAND $<TARGET_FILE:json2json> ${EXAMPLE_JSON})
set_property(TEST json2json APPEND PROPERTY LABELS acceptance per_implementation)
tests/compilation_failure_tests/CMakeLists.txt
+2 -2
View File
@@ -8,8 +8,8 @@
# adds a compilation test. Two targets are created, one expected to
# succeed compilation and one that is expected to fail.
function(add_dual_compile_test TEST_NAME)
add_cpp_test(${TEST_NAME}_should_compile SOURCES ${TEST_NAME}.cpp COMPILE_ONLY)
add_cpp_test(${TEST_NAME}_should_not_compile SOURCES ${TEST_NAME}.cpp COMPILE_ONLY WILL_FAIL LABELS acceptance)
add_cpp_test(${TEST_NAME}_should_compile SOURCES ${TEST_NAME}.cpp COMPILE_ONLY LABELS no_mingw)
add_cpp_test(${TEST_NAME}_should_not_compile SOURCES ${TEST_NAME}.cpp COMPILE_ONLY WILL_FAIL LABELS acceptance no_mingw)
target_compile_definitions(${TEST_NAME}_should_not_compile PRIVATE COMPILATION_TEST_USE_FAILING_CODE=1)
endfunction(add_dual_compile_test)
tests/ondemand/CMakeLists.txt
+20 -2
View File
@@ -1,7 +1,25 @@
# All remaining tests link with simdjson proper
link_libraries(simdjson)
include_directories(..)
add_cpp_test(ondemand_basictests LABELS acceptance per_implementation)
add_cpp_test(ondemand_active_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_compilation_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_dom_api_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_error_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_key_string_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_number_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_ordering_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_parse_api_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_twitter_tests LABELS ondemand acceptance per_implementation)
if(HAVE_POSIX_FORK AND HAVE_POSIX_WAIT) # assert tests use fork and wait, which aren't on MSVC
add_cpp_test(ondemand_assert_out_of_order_values LABELS per_implementation explicitonly assert)
add_cpp_test(ondemand_assert_out_of_order_values LABELS assert per_implementation explicitonly ondemand)
endif()
# Copy the simdjson dll into the tests directory
if(MSVC)
add_custom_command(TARGET ondemand_dom_api_tests POST_BUILD # Adds a post-build event
COMMAND ${CMAKE_COMMAND} -E copy_if_different # which executes "cmake -E copy_if_different..."
"$<TARGET_FILE:simdjson>" # <--this is in-file
"$<TARGET_FILE_DIR:ondemand_dom_api_tests>") # <--this is out-file path
endif(MSVC)
tests/ondemand/ondemand_active_tests.cpp
+85
View File
@@ -0,0 +1,85 @@
#include <cinttypes>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <string>
#include <vector>
#include <cmath>
#include <set>
#include <sstream>
#include <utility>
#include <unistd.h>
#include "simdjson.h"
#include "test_ondemand.h"
using namespace simdjson;
using namespace simdjson::builtin;
namespace active_tests {
#if SIMDJSON_EXCEPTIONS
bool parser_child() {
TEST_START();
ondemand::parser parser;
const padded_string json = R"({ "parent": {"child1": {"name": "John"} , "child2": {"name": "Daniel"}} })"_padded;
auto doc = parser.iterate(json);
ondemand::object parent = doc["parent"];
{
ondemand::object c1 = parent["child1"];
if(std::string_view(c1["name"]) != "John") { return false; }
}
{
ondemand::object c2 = parent["child2"];
if(std::string_view(c2["name"]) != "Daniel") { return false; }
}
return true;
}
bool parser_doc_correct() {
TEST_START();
ondemand::parser parser;
const padded_string json = R"({ "key1": 1, "key2":2, "key3": 3 })"_padded;
auto doc = parser.iterate(json);
ondemand::object root_object = doc.get_object();
int64_t k1 = root_object["key1"];
int64_t k2 = root_object["key2"];
int64_t k3 = root_object["key3"];
return (k1 == 1) && (k2 == 2) && (k3 == 3);
}
bool parser_doc_limits() {
TEST_START();
ondemand::parser parser;
const padded_string json = R"({ "key1": 1, "key2":2, "key3": 3 })"_padded;
auto doc = parser.iterate(json);
int64_t k1 = doc["key1"];
try {
int64_t k2 = doc["key2"];
(void) k2;
} catch (simdjson::simdjson_error &) {
return true; // we expect to fail.
}
(void) k1;
return false;
}
#endif // SIMDJSON_EXCEPTIONS
bool run() {
return
#if SIMDJSON_EXCEPTIONS
parser_child() &&
parser_doc_correct() &&
// parser_doc_limits() && // Failure is dependent on build type here ...
#endif // SIMDJSON_EXCEPTIONS
true;
}
} // namespace active_tests
int main(int argc, char *argv[]) {
return test_main(argc, argv, active_tests::run);
}
tests/ondemand/ondemand_basictests.cpp
-2139
View File
File diff suppressed because it is too large Load Diff
tests/ondemand/ondemand_compilation_tests.cpp
+80
View File
@@ -0,0 +1,80 @@
#include <cinttypes>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <string>
#include <vector>
#include <cmath>
#include <set>
#include <sstream>
#include <utility>
#include <unistd.h>
#include "simdjson.h"
#include "test_ondemand.h"
using namespace simdjson;
using namespace simdjson::builtin;
#if SIMDJSON_EXCEPTIONS
// bogus functions for compilation tests
void process1(int ) {}
void process2(int ) {}
void process3(int ) {}
// Do not run this, it is only meant to compile
void compilation_test_1() {
const padded_string bogus = ""_padded;
ondemand::parser parser;
auto doc = parser.iterate(bogus);
for (ondemand::object my_object : doc["mykey"]) {
for (auto field : my_object) {
if (field.key() == "key_value1") { process1(field.value()); }
else if (field.key() == "key_value2") { process2(field.value()); }
else if (field.key() == "key_value3") { process3(field.value()); }
}
}
}
// Do not run this, it is only meant to compile
void compilation_test_2() {
const padded_string bogus = ""_padded;
ondemand::parser parser;
auto doc = parser.iterate(bogus);
std::set<std::string_view> default_users;
ondemand::array tweets = doc["statuses"].get_array();
for (auto tweet_value : tweets) {
auto tweet = tweet_value.get_object();
ondemand::object user = tweet["user"].get_object();
std::string_view screen_name = user["screen_name"].get_string();
bool default_profile = user["default_profile"].get_bool();
if (default_profile) { default_users.insert(screen_name); }
}
}
// Do not run this, it is only meant to compile
void compilation_test_3() {
const padded_string bogus = ""_padded;
ondemand::parser parser;
auto doc = parser.iterate(bogus);
ondemand::array tweets;
if(! doc["statuses"].get(tweets)) { return; }
for (auto tweet_value : tweets) {
auto tweet = tweet_value.get_object();
for (auto field : tweet) {
std::string_view key = field.unescaped_key().value();
std::cout << "key = " << key << std::endl;
std::string_view val = std::string_view(field.value());
std::cout << "value (assuming it is a string) = " << val << std::endl;
}
}
}
#endif // SIMDJSON_EXCEPTIONS
int main(void) {
return 0;
}
tests/ondemand/ondemand_dom_api_tests.cpp
+1208
View File
File diff suppressed because it is too large Load Diff
tests/ondemand/ondemand_error_tests.cpp
+438
View File
@@ -0,0 +1,438 @@
#include <cinttypes>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <string>
#include <vector>
#include <cmath>
#include <set>
#include <sstream>
#include <utility>
#include <unistd.h>
#include "simdjson.h"
#include "test_ondemand.h"
using namespace simdjson;
using namespace simdjson::builtin;
namespace error_tests {
using namespace std;
bool empty_document_error() {
TEST_START();
ondemand::parser parser;
ASSERT_ERROR( parser.iterate(""_padded), EMPTY );
TEST_SUCCEED();
}
namespace wrong_type {
#define TEST_CAST_ERROR(JSON, TYPE, ERROR) \
std::cout << "- Subtest: get_" << (#TYPE) << "() - JSON: " << (JSON) << std::endl; \
if (!test_ondemand_doc((JSON##_padded), [&](auto doc_result) { \
ASSERT_ERROR( doc_result.get_##TYPE(), (ERROR) ); \
return true; \
})) { \
return false; \
} \
{ \
padded_string a_json(std::string(R"({ "a": )") + JSON + " })"); \
std::cout << R"(- Subtest: get_)" << (#TYPE) << "() - JSON: " << a_json << std::endl; \
if (!test_ondemand_doc(a_json, [&](auto doc_result) { \
ASSERT_ERROR( doc_result["a"].get_##TYPE(), (ERROR) ); \
return true; \
})) { \
return false; \
}; \
}
bool wrong_type_array() {
TEST_START();
TEST_CAST_ERROR("[]", object, INCORRECT_TYPE);
TEST_CAST_ERROR("[]", bool, INCORRECT_TYPE);
TEST_CAST_ERROR("[]", int64, NUMBER_ERROR);
TEST_CAST_ERROR("[]", uint64, NUMBER_ERROR);
TEST_CAST_ERROR("[]", double, NUMBER_ERROR);
TEST_CAST_ERROR("[]", string, INCORRECT_TYPE);
TEST_CAST_ERROR("[]", raw_json_string, INCORRECT_TYPE);
TEST_SUCCEED();
}
bool wrong_type_object() {
TEST_START();
TEST_CAST_ERROR("{}", array, INCORRECT_TYPE);
TEST_CAST_ERROR("{}", bool, INCORRECT_TYPE);
TEST_CAST_ERROR("{}", int64, NUMBER_ERROR);
TEST_CAST_ERROR("{}", uint64, NUMBER_ERROR);
TEST_CAST_ERROR("{}", double, NUMBER_ERROR);
TEST_CAST_ERROR("{}", string, INCORRECT_TYPE);
TEST_CAST_ERROR("{}", raw_json_string, INCORRECT_TYPE);
TEST_SUCCEED();
}
bool wrong_type_true() {
TEST_START();
TEST_CAST_ERROR("true", array, INCORRECT_TYPE);
TEST_CAST_ERROR("true", object, INCORRECT_TYPE);
TEST_CAST_ERROR("true", int64, NUMBER_ERROR);
TEST_CAST_ERROR("true", uint64, NUMBER_ERROR);
TEST_CAST_ERROR("true", double, NUMBER_ERROR);
TEST_CAST_ERROR("true", string, INCORRECT_TYPE);
TEST_CAST_ERROR("true", raw_json_string, INCORRECT_TYPE);
TEST_SUCCEED();
}
bool wrong_type_false() {
TEST_START();
TEST_CAST_ERROR("false", array, INCORRECT_TYPE);
TEST_CAST_ERROR("false", object, INCORRECT_TYPE);
TEST_CAST_ERROR("false", int64, NUMBER_ERROR);
TEST_CAST_ERROR("false", uint64, NUMBER_ERROR);
TEST_CAST_ERROR("false", double, NUMBER_ERROR);
TEST_CAST_ERROR("false", string, INCORRECT_TYPE);
TEST_CAST_ERROR("false", raw_json_string, INCORRECT_TYPE);
TEST_SUCCEED();
}
bool wrong_type_null() {
TEST_START();
TEST_CAST_ERROR("null", array, INCORRECT_TYPE);
TEST_CAST_ERROR("null", object, INCORRECT_TYPE);
TEST_CAST_ERROR("null", int64, NUMBER_ERROR);
TEST_CAST_ERROR("null", uint64, NUMBER_ERROR);
TEST_CAST_ERROR("null", double, NUMBER_ERROR);
TEST_CAST_ERROR("null", string, INCORRECT_TYPE);
TEST_CAST_ERROR("null", raw_json_string, INCORRECT_TYPE);
TEST_SUCCEED();
}
bool wrong_type_1() {
TEST_START();
TEST_CAST_ERROR("1", array, INCORRECT_TYPE);
TEST_CAST_ERROR("1", object, INCORRECT_TYPE);
TEST_CAST_ERROR("1", bool, INCORRECT_TYPE);
TEST_CAST_ERROR("1", string, INCORRECT_TYPE);
TEST_CAST_ERROR("1", raw_json_string, INCORRECT_TYPE);
TEST_SUCCEED();
}
bool wrong_type_negative_1() {
TEST_START();
TEST_CAST_ERROR("-1", array, INCORRECT_TYPE);
TEST_CAST_ERROR("-1", object, INCORRECT_TYPE);
TEST_CAST_ERROR("-1", bool, INCORRECT_TYPE);
TEST_CAST_ERROR("-1", uint64, NUMBER_ERROR);
TEST_CAST_ERROR("-1", string, INCORRECT_TYPE);
TEST_CAST_ERROR("-1", raw_json_string, INCORRECT_TYPE);
TEST_SUCCEED();
}
bool wrong_type_float() {
TEST_START();
TEST_CAST_ERROR("1.1", array, INCORRECT_TYPE);
TEST_CAST_ERROR("1.1", object, INCORRECT_TYPE);
TEST_CAST_ERROR("1.1", bool, INCORRECT_TYPE);
TEST_CAST_ERROR("1.1", int64, NUMBER_ERROR);
TEST_CAST_ERROR("1.1", uint64, NUMBER_ERROR);
TEST_CAST_ERROR("1.1", string, INCORRECT_TYPE);
TEST_CAST_ERROR("1.1", raw_json_string, INCORRECT_TYPE);
TEST_SUCCEED();
}
bool wrong_type_negative_int64_overflow() {
TEST_START();
TEST_CAST_ERROR("-9223372036854775809", array, INCORRECT_TYPE);
TEST_CAST_ERROR("-9223372036854775809", object, INCORRECT_TYPE);
TEST_CAST_ERROR("-9223372036854775809", bool, INCORRECT_TYPE);
TEST_CAST_ERROR("-9223372036854775809", int64, NUMBER_ERROR);
TEST_CAST_ERROR("-9223372036854775809", uint64, NUMBER_ERROR);
TEST_CAST_ERROR("-9223372036854775809", string, INCORRECT_TYPE);
TEST_CAST_ERROR("-9223372036854775809", raw_json_string, INCORRECT_TYPE);
TEST_SUCCEED();
}
bool wrong_type_int64_overflow() {
TEST_START();
TEST_CAST_ERROR("9223372036854775808", array, INCORRECT_TYPE);
TEST_CAST_ERROR("9223372036854775808", object, INCORRECT_TYPE);
TEST_CAST_ERROR("9223372036854775808", bool, INCORRECT_TYPE);
// TODO BUG: this should be an error but is presently not
// TEST_CAST_ERROR("9223372036854775808", int64, NUMBER_ERROR);
TEST_CAST_ERROR("9223372036854775808", string, INCORRECT_TYPE);
TEST_CAST_ERROR("9223372036854775808", raw_json_string, INCORRECT_TYPE);
TEST_SUCCEED();
}
bool wrong_type_uint64_overflow() {
TEST_START();
TEST_CAST_ERROR("18446744073709551616", array, INCORRECT_TYPE);
TEST_CAST_ERROR("18446744073709551616", object, INCORRECT_TYPE);
TEST_CAST_ERROR("18446744073709551616", bool, INCORRECT_TYPE);
TEST_CAST_ERROR("18446744073709551616", int64, NUMBER_ERROR);
// TODO BUG: this should be an error but is presently not
// TEST_CAST_ERROR("18446744073709551616", uint64, NUMBER_ERROR);
TEST_CAST_ERROR("18446744073709551616", string, INCORRECT_TYPE);
TEST_CAST_ERROR("18446744073709551616", raw_json_string, INCORRECT_TYPE);
TEST_SUCCEED();
}
bool run() {
return
wrong_type_1() &&
wrong_type_array() &&
wrong_type_false() &&
wrong_type_float() &&
wrong_type_int64_overflow() &&
wrong_type_negative_1() &&
wrong_type_negative_int64_overflow() &&
wrong_type_null() &&
wrong_type_object() &&
wrong_type_true() &&
wrong_type_uint64_overflow() &&
true;
}
} // namespace wrong_type
template<typename V, typename T>
bool assert_iterate(T array, V *expected, size_t N, simdjson::error_code *expected_error, size_t N2) {
size_t count = 0;
for (auto elem : std::forward<T>(array)) {
V actual;
auto actual_error = elem.get(actual);
if (count >= N) {
if (count >= (N+N2)) {
std::cerr << "FAIL: Extra error reported: " << actual_error << std::endl;
return false;
}
ASSERT_ERROR(actual_error, expected_error[count - N]);
} else {
ASSERT_SUCCESS(actual_error);
ASSERT_EQUAL(actual, expected[count]);
}
count++;
}
ASSERT_EQUAL(count, N+N2);
return true;
}
template<typename V, size_t N, size_t N2, typename T>
bool assert_iterate(T &array, V (&&expected)[N], simdjson::error_code (&&expected_error)[N2]) {
return assert_iterate<V, T&>(array, expected, N, expected_error, N2);
}
template<size_t N2, typename T>
bool assert_iterate(T &array, simdjson::error_code (&&expected_error)[N2]) {
return assert_iterate<int64_t, T&>(array, nullptr, 0, expected_error, N2);
}
template<typename V, size_t N, typename T>
bool assert_iterate(T &array, V (&&expected)[N]) {
return assert_iterate<V, T&&>(array, expected, N, nullptr, 0);
}
template<typename V, size_t N, size_t N2, typename T>
bool assert_iterate(T &&array, V (&&expected)[N], simdjson::error_code (&&expected_error)[N2]) {
return assert_iterate<V, T&&>(std::forward<T>(array), expected, N, expected_error, N2);
}
template<size_t N2, typename T>
bool assert_iterate(T &&array, simdjson::error_code (&&expected_error)[N2]) {
return assert_iterate<int64_t, T&&>(std::forward<T>(array), nullptr, 0, expected_error, N2);
}
template<typename V, size_t N, typename T>
bool assert_iterate(T &&array, V (&&expected)[N]) {
return assert_iterate<V, T&&>(std::forward<T>(array), expected, N, nullptr, 0);
}
bool top_level_array_iterate_error() {
TEST_START();
ONDEMAND_SUBTEST("missing comma", "[1 1]", assert_iterate(doc, { int64_t(1) }, { TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", "[1,,1]", assert_iterate(doc, { int64_t(1) }, { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", "[,]", assert_iterate(doc, { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", "[,,]", assert_iterate(doc, { NUMBER_ERROR, TAPE_ERROR }));
TEST_SUCCEED();
}
bool top_level_array_iterate_unclosed_error() {
TEST_START();
ONDEMAND_SUBTEST("unclosed extra comma", "[,", assert_iterate(doc, { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed ", "[1 ", assert_iterate(doc, { int64_t(1) }, { TAPE_ERROR }));
// TODO These pass the user values that may run past the end of the buffer if they aren't careful
// In particular, if the padding is decorated with the wrong values, we could cause overrun!
ONDEMAND_SUBTEST("unclosed extra comma", "[,,", assert_iterate(doc, { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed ", "[1,", assert_iterate(doc, { int64_t(1) }, { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed ", "[1", assert_iterate(doc, { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed ", "[", assert_iterate(doc, { NUMBER_ERROR, TAPE_ERROR }));
TEST_SUCCEED();
}
bool array_iterate_error() {
TEST_START();
ONDEMAND_SUBTEST("missing comma", R"({ "a": [1 1] })", assert_iterate(doc["a"], { int64_t(1) }, { TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [1,,1] })", assert_iterate(doc["a"], { int64_t(1) }, { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [1,,] })", assert_iterate(doc["a"], { int64_t(1) }, { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [,] })", assert_iterate(doc["a"], { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [,,] })", assert_iterate(doc["a"], { NUMBER_ERROR, TAPE_ERROR }));
TEST_SUCCEED();
}
bool array_iterate_unclosed_error() {
TEST_START();
ONDEMAND_SUBTEST("unclosed extra comma", R"({ "a": [,)", assert_iterate(doc["a"], { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed extra comma", R"({ "a": [,,)", assert_iterate(doc["a"], { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1 )", assert_iterate(doc["a"], { int64_t(1) }, { TAPE_ERROR }));
// TODO These pass the user values that may run past the end of the buffer if they aren't careful
// In particular, if the padding is decorated with the wrong values, we could cause overrun!
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1,)", assert_iterate(doc["a"], { int64_t(1) }, { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [1)", assert_iterate(doc["a"], { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed ", R"({ "a": [)", assert_iterate(doc["a"], { NUMBER_ERROR, TAPE_ERROR }));
TEST_SUCCEED();
}
template<typename V, typename T>
bool assert_iterate_object(T &&object, const char **expected_key, V *expected, size_t N, simdjson::error_code *expected_error, size_t N2) {
size_t count = 0;
for (auto field : object) {
V actual;
auto actual_error = field.value().get(actual);
if (count >= N) {
ASSERT((count - N) < N2, "Extra error reported");
ASSERT_ERROR(actual_error, expected_error[count - N]);
} else {
ASSERT_SUCCESS(actual_error);
ASSERT_EQUAL(field.key().first, expected_key[count]);
ASSERT_EQUAL(actual, expected[count]);
}
count++;
}
ASSERT_EQUAL(count, N+N2);
return true;
}
template<typename V, size_t N, size_t N2, typename T>
bool assert_iterate_object(T &&object, const char *(&&expected_key)[N], V (&&expected)[N], simdjson::error_code (&&expected_error)[N2]) {
return assert_iterate_object<V, T>(std::forward<T>(object), expected_key, expected, N, expected_error, N2);
}
template<size_t N2, typename T>
bool assert_iterate_object(T &&object, simdjson::error_code (&&expected_error)[N2]) {
return assert_iterate_object<int64_t, T>(std::forward<T>(object), nullptr, nullptr, 0, expected_error, N2);
}
template<typename V, size_t N, typename T>
bool assert_iterate_object(T &&object, const char *(&&expected_key)[N], V (&&expected)[N]) {
return assert_iterate_object<V, T>(std::forward<T>(object), expected_key, expected, N, nullptr, 0);
}
bool object_iterate_error() {
TEST_START();
ONDEMAND_SUBTEST("missing colon", R"({ "a" 1, "b": 2 })", assert_iterate_object(doc.get_object(), { TAPE_ERROR }));
ONDEMAND_SUBTEST("missing key ", R"({ : 1, "b": 2 })", assert_iterate_object(doc.get_object(), { TAPE_ERROR }));
ONDEMAND_SUBTEST("missing value", R"({ "a": , "b": 2 })", assert_iterate_object(doc.get_object(), { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("missing comma", R"({ "a": 1 "b": 2 })", assert_iterate_object(doc.get_object(), { "a" }, { int64_t(1) }, { TAPE_ERROR }));
TEST_SUCCEED();
}
bool object_iterate_wrong_key_type_error() {
TEST_START();
ONDEMAND_SUBTEST("wrong key type", R"({ 1: 1, "b": 2 })", assert_iterate_object(doc.get_object(), { TAPE_ERROR }));
ONDEMAND_SUBTEST("wrong key type", R"({ true: 1, "b": 2 })", assert_iterate_object(doc.get_object(), { TAPE_ERROR }));
ONDEMAND_SUBTEST("wrong key type", R"({ false: 1, "b": 2 })", assert_iterate_object(doc.get_object(), { TAPE_ERROR }));
ONDEMAND_SUBTEST("wrong key type", R"({ null: 1, "b": 2 })", assert_iterate_object(doc.get_object(), { TAPE_ERROR }));
ONDEMAND_SUBTEST("wrong key type", R"({ []: 1, "b": 2 })", assert_iterate_object(doc.get_object(), { TAPE_ERROR }));
ONDEMAND_SUBTEST("wrong key type", R"({ {}: 1, "b": 2 })", assert_iterate_object(doc.get_object(), { TAPE_ERROR }));
TEST_SUCCEED();
}
bool object_iterate_unclosed_error() {
TEST_START();
ONDEMAND_SUBTEST("unclosed", R"({ "a": 1, )", assert_iterate_object(doc.get_object(), { "a" }, { int64_t(1) }, { TAPE_ERROR }));
// TODO These next two pass the user a value that may run past the end of the buffer if they aren't careful.
// In particular, if the padding is decorated with the wrong values, we could cause overrun!
ONDEMAND_SUBTEST("unclosed", R"({ "a": 1 )", assert_iterate_object(doc.get_object(), { "a" }, { int64_t(1) }, { TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_iterate_object(doc.get_object(), { NUMBER_ERROR, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed", R"({ "a" )", assert_iterate_object(doc.get_object(), { TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed", R"({ )", assert_iterate_object(doc.get_object(), { TAPE_ERROR }));
TEST_SUCCEED();
}
bool object_lookup_error() {
TEST_START();
ONDEMAND_SUBTEST("missing colon", R"({ "a" 1, "b": 2 })", assert_error(doc["a"], TAPE_ERROR));
ONDEMAND_SUBTEST("missing key ", R"({ : 1, "b": 2 })", assert_error(doc["a"], TAPE_ERROR));
ONDEMAND_SUBTEST("missing value", R"({ "a": , "b": 2 })", assert_success(doc["a"]));
ONDEMAND_SUBTEST("missing comma", R"({ "a": 1 "b": 2 })", assert_success(doc["a"]));
TEST_SUCCEED();
}
bool object_lookup_unclosed_error() {
TEST_START();
// TODO This one passes the user a value that may run past the end of the buffer if they aren't careful.
// In particular, if the padding is decorated with the wrong values, we could cause overrun!
ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_success(doc["a"]));
ONDEMAND_SUBTEST("unclosed", R"({ "a" )", assert_error(doc["a"], TAPE_ERROR));
ONDEMAND_SUBTEST("unclosed", R"({ )", assert_error(doc["a"], TAPE_ERROR));
TEST_SUCCEED();
}
bool object_lookup_miss_error() {
TEST_START();
ONDEMAND_SUBTEST("missing colon", R"({ "a" 1, "b": 2 })", assert_error(doc["b"], TAPE_ERROR));
ONDEMAND_SUBTEST("missing key ", R"({ : 1, "b": 2 })", assert_error(doc["b"], TAPE_ERROR));
ONDEMAND_SUBTEST("missing value", R"({ "a": , "b": 2 })", assert_error(doc["b"], TAPE_ERROR));
ONDEMAND_SUBTEST("missing comma", R"({ "a": 1 "b": 2 })", assert_error(doc["b"], TAPE_ERROR));
TEST_SUCCEED();
}
bool object_lookup_miss_wrong_key_type_error() {
TEST_START();
ONDEMAND_SUBTEST("wrong key type", R"({ 1: 1, "b": 2 })", assert_error(doc["b"], TAPE_ERROR));
ONDEMAND_SUBTEST("wrong key type", R"({ true: 1, "b": 2 })", assert_error(doc["b"], TAPE_ERROR));
ONDEMAND_SUBTEST("wrong key type", R"({ false: 1, "b": 2 })", assert_error(doc["b"], TAPE_ERROR));
ONDEMAND_SUBTEST("wrong key type", R"({ null: 1, "b": 2 })", assert_error(doc["b"], TAPE_ERROR));
ONDEMAND_SUBTEST("wrong key type", R"({ []: 1, "b": 2 })", assert_error(doc["b"], TAPE_ERROR));
ONDEMAND_SUBTEST("wrong key type", R"({ {}: 1, "b": 2 })", assert_error(doc["b"], TAPE_ERROR));
TEST_SUCCEED();
}
bool object_lookup_miss_unclosed_error() {
TEST_START();
ONDEMAND_SUBTEST("unclosed", R"({ "a": 1, )", assert_error(doc["b"], TAPE_ERROR));
// TODO These next two pass the user a value that may run past the end of the buffer if they aren't careful.
// In particular, if the padding is decorated with the wrong values, we could cause overrun!
ONDEMAND_SUBTEST("unclosed", R"({ "a": 1 )", assert_error(doc["b"], TAPE_ERROR));
ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_error(doc["b"], TAPE_ERROR));
ONDEMAND_SUBTEST("unclosed", R"({ "a" )", assert_error(doc["b"], TAPE_ERROR));
ONDEMAND_SUBTEST("unclosed", R"({ )", assert_error(doc["b"], TAPE_ERROR));
TEST_SUCCEED();
}
bool object_lookup_miss_next_error() {
TEST_START();
ONDEMAND_SUBTEST("missing comma", R"({ "a": 1 "b": 2 })", ([&]() {
auto obj = doc.get_object();
return assert_result<int64_t>(obj["a"], 1) && assert_error(obj["b"], TAPE_ERROR);
})());
TEST_SUCCEED();
}
bool run() {
return
empty_document_error() &&
top_level_array_iterate_error() &&
top_level_array_iterate_unclosed_error() &&
array_iterate_error() &&
array_iterate_unclosed_error() &&
wrong_type::run() &&
object_iterate_error() &&
object_iterate_wrong_key_type_error() &&
object_iterate_unclosed_error() &&
object_lookup_error() &&
object_lookup_unclosed_error() &&
object_lookup_miss_error() &&
object_lookup_miss_unclosed_error() &&
object_lookup_miss_wrong_key_type_error() &&
object_lookup_miss_next_error() &&
true;
}
}
int main(int argc, char *argv[]) {
return test_main(argc, argv, error_tests::run);
}
tests/ondemand/ondemand_key_string_tests.cpp
+47
View File
@@ -0,0 +1,47 @@
#include <cinttypes>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <string>
#include <vector>
#include <cmath>
#include <set>
#include <sstream>
#include <utility>
#include <unistd.h>
#include "simdjson.h"
#include "test_ondemand.h"
using namespace simdjson;
using namespace simdjson::builtin;
namespace key_string_tests {
#if SIMDJSON_EXCEPTIONS
bool parser_key_value() {
TEST_START();
ondemand::parser parser;
const padded_string json = R"({ "1": "1", "2": "2", "3": "3", "abc": "abc", "\u0075": "\u0075" })"_padded;
auto doc = parser.iterate(json);
for(auto field : doc.get_object()) {
std::string_view keyv = field.unescaped_key();
std::string_view valuev = field.value();
if(keyv != valuev) { return false; }
}
return true;
}
#endif // SIMDJSON_EXCEPTIONS
bool run() {
return
#if SIMDJSON_EXCEPTIONS
parser_key_value() &&
#endif // SIMDJSON_EXCEPTIONS
true;
}
} // namespace key_string_tests
int main(int argc, char *argv[]) {
return test_main(argc, argv, key_string_tests::run);
}
tests/ondemand/ondemand_number_tests.cpp
+203
View File
@@ -0,0 +1,203 @@
#include <cinttypes>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <string>
#include <vector>
#include <cmath>
#include <set>
#include <sstream>
#include <utility>
#include <unistd.h>
#include "simdjson.h"
#include "test_ondemand.h"
using namespace simdjson;
using namespace simdjson::builtin;
namespace number_tests {
bool small_integers() {
std::cout << __func__ << std::endl;
for (int64_t m = 10; m < 20; m++) {
for (int64_t i = -1024; i < 1024; i++) {
if(!test_ondemand<int64_t>(std::to_string(i),
[&](int64_t actual) {
ASSERT_EQUAL(actual, i);
return true;
})) {
return false;
} // if
} // for i
} // for m
return true;
}
bool powers_of_two() {
std::cout << __func__ << std::endl;
// converts the double "expected" to a padded string
auto format_into_padded=[](const double expected) -> padded_string
{
char buf[1024];
const auto n = std::snprintf(buf,
sizeof(buf),
"%.*e",
std::numeric_limits<double>::max_digits10 - 1,
expected);
const auto nz=static_cast<size_t>(n);
if (n<0 || nz >= sizeof(buf)) { std::abort(); }
return padded_string(buf, nz);
};
for (int i = -1075; i < 1024; ++i) {// large negative values should be zero.
const double expected = std::pow(2, i);
const auto buf=format_into_padded(expected);
std::fflush(nullptr);
if(!test_ondemand<double>(buf,
[&](double actual) {
if(actual!=expected) {
std::cerr << "JSON '" << buf << " parsed to ";
std::fprintf( stderr," %18.18g instead of %18.18g\n", actual, expected); // formatting numbers is easier with printf
SIMDJSON_SHOW_DEFINE(FLT_EVAL_METHOD);
return false;
}
return true;
})) {
return false;
} // if
} // for i
return true;
}
static const double testing_power_of_ten[] = {
1e-307, 1e-306, 1e-305, 1e-304, 1e-303, 1e-302, 1e-301, 1e-300, 1e-299,
1e-298, 1e-297, 1e-296, 1e-295, 1e-294, 1e-293, 1e-292, 1e-291, 1e-290,
1e-289, 1e-288, 1e-287, 1e-286, 1e-285, 1e-284, 1e-283, 1e-282, 1e-281,
1e-280, 1e-279, 1e-278, 1e-277, 1e-276, 1e-275, 1e-274, 1e-273, 1e-272,
1e-271, 1e-270, 1e-269, 1e-268, 1e-267, 1e-266, 1e-265, 1e-264, 1e-263,
1e-262, 1e-261, 1e-260, 1e-259, 1e-258, 1e-257, 1e-256, 1e-255, 1e-254,
1e-253, 1e-252, 1e-251, 1e-250, 1e-249, 1e-248, 1e-247, 1e-246, 1e-245,
1e-244, 1e-243, 1e-242, 1e-241, 1e-240, 1e-239, 1e-238, 1e-237, 1e-236,
1e-235, 1e-234, 1e-233, 1e-232, 1e-231, 1e-230, 1e-229, 1e-228, 1e-227,
1e-226, 1e-225, 1e-224, 1e-223, 1e-222, 1e-221, 1e-220, 1e-219, 1e-218,
1e-217, 1e-216, 1e-215, 1e-214, 1e-213, 1e-212, 1e-211, 1e-210, 1e-209,
1e-208, 1e-207, 1e-206, 1e-205, 1e-204, 1e-203, 1e-202, 1e-201, 1e-200,
1e-199, 1e-198, 1e-197, 1e-196, 1e-195, 1e-194, 1e-193, 1e-192, 1e-191,
1e-190, 1e-189, 1e-188, 1e-187, 1e-186, 1e-185, 1e-184, 1e-183, 1e-182,
1e-181, 1e-180, 1e-179, 1e-178, 1e-177, 1e-176, 1e-175, 1e-174, 1e-173,
1e-172, 1e-171, 1e-170, 1e-169, 1e-168, 1e-167, 1e-166, 1e-165, 1e-164,
1e-163, 1e-162, 1e-161, 1e-160, 1e-159, 1e-158, 1e-157, 1e-156, 1e-155,
1e-154, 1e-153, 1e-152, 1e-151, 1e-150, 1e-149, 1e-148, 1e-147, 1e-146,
1e-145, 1e-144, 1e-143, 1e-142, 1e-141, 1e-140, 1e-139, 1e-138, 1e-137,
1e-136, 1e-135, 1e-134, 1e-133, 1e-132, 1e-131, 1e-130, 1e-129, 1e-128,
1e-127, 1e-126, 1e-125, 1e-124, 1e-123, 1e-122, 1e-121, 1e-120, 1e-119,
1e-118, 1e-117, 1e-116, 1e-115, 1e-114, 1e-113, 1e-112, 1e-111, 1e-110,
1e-109, 1e-108, 1e-107, 1e-106, 1e-105, 1e-104, 1e-103, 1e-102, 1e-101,
1e-100, 1e-99, 1e-98, 1e-97, 1e-96, 1e-95, 1e-94, 1e-93, 1e-92,
1e-91, 1e-90, 1e-89, 1e-88, 1e-87, 1e-86, 1e-85, 1e-84, 1e-83,
1e-82, 1e-81, 1e-80, 1e-79, 1e-78, 1e-77, 1e-76, 1e-75, 1e-74,
1e-73, 1e-72, 1e-71, 1e-70, 1e-69, 1e-68, 1e-67, 1e-66, 1e-65,
1e-64, 1e-63, 1e-62, 1e-61, 1e-60, 1e-59, 1e-58, 1e-57, 1e-56,
1e-55, 1e-54, 1e-53, 1e-52, 1e-51, 1e-50, 1e-49, 1e-48, 1e-47,
1e-46, 1e-45, 1e-44, 1e-43, 1e-42, 1e-41, 1e-40, 1e-39, 1e-38,
1e-37, 1e-36, 1e-35, 1e-34, 1e-33, 1e-32, 1e-31, 1e-30, 1e-29,
1e-28, 1e-27, 1e-26, 1e-25, 1e-24, 1e-23, 1e-22, 1e-21, 1e-20,
1e-19, 1e-18, 1e-17, 1e-16, 1e-15, 1e-14, 1e-13, 1e-12, 1e-11,
1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2,
1e-1, 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7,
1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16,
1e17, 1e18, 1e19, 1e20, 1e21, 1e22, 1e23, 1e24, 1e25,
1e26, 1e27, 1e28, 1e29, 1e30, 1e31, 1e32, 1e33, 1e34,
1e35, 1e36, 1e37, 1e38, 1e39, 1e40, 1e41, 1e42, 1e43,
1e44, 1e45, 1e46, 1e47, 1e48, 1e49, 1e50, 1e51, 1e52,
1e53, 1e54, 1e55, 1e56, 1e57, 1e58, 1e59, 1e60, 1e61,
1e62, 1e63, 1e64, 1e65, 1e66, 1e67, 1e68, 1e69, 1e70,
1e71, 1e72, 1e73, 1e74, 1e75, 1e76, 1e77, 1e78, 1e79,
1e80, 1e81, 1e82, 1e83, 1e84, 1e85, 1e86, 1e87, 1e88,
1e89, 1e90, 1e91, 1e92, 1e93, 1e94, 1e95, 1e96, 1e97,
1e98, 1e99, 1e100, 1e101, 1e102, 1e103, 1e104, 1e105, 1e106,
1e107, 1e108, 1e109, 1e110, 1e111, 1e112, 1e113, 1e114, 1e115,
1e116, 1e117, 1e118, 1e119, 1e120, 1e121, 1e122, 1e123, 1e124,
1e125, 1e126, 1e127, 1e128, 1e129, 1e130, 1e131, 1e132, 1e133,
1e134, 1e135, 1e136, 1e137, 1e138, 1e139, 1e140, 1e141, 1e142,
1e143, 1e144, 1e145, 1e146, 1e147, 1e148, 1e149, 1e150, 1e151,
1e152, 1e153, 1e154, 1e155, 1e156, 1e157, 1e158, 1e159, 1e160,
1e161, 1e162, 1e163, 1e164, 1e165, 1e166, 1e167, 1e168, 1e169,
1e170, 1e171, 1e172, 1e173, 1e174, 1e175, 1e176, 1e177, 1e178,
1e179, 1e180, 1e181, 1e182, 1e183, 1e184, 1e185, 1e186, 1e187,
1e188, 1e189, 1e190, 1e191, 1e192, 1e193, 1e194, 1e195, 1e196,
1e197, 1e198, 1e199, 1e200, 1e201, 1e202, 1e203, 1e204, 1e205,
1e206, 1e207, 1e208, 1e209, 1e210, 1e211, 1e212, 1e213, 1e214,
1e215, 1e216, 1e217, 1e218, 1e219, 1e220, 1e221, 1e222, 1e223,
1e224, 1e225, 1e226, 1e227, 1e228, 1e229, 1e230, 1e231, 1e232,
1e233, 1e234, 1e235, 1e236, 1e237, 1e238, 1e239, 1e240, 1e241,
1e242, 1e243, 1e244, 1e245, 1e246, 1e247, 1e248, 1e249, 1e250,
1e251, 1e252, 1e253, 1e254, 1e255, 1e256, 1e257, 1e258, 1e259,
1e260, 1e261, 1e262, 1e263, 1e264, 1e265, 1e266, 1e267, 1e268,
1e269, 1e270, 1e271, 1e272, 1e273, 1e274, 1e275, 1e276, 1e277,
1e278, 1e279, 1e280, 1e281, 1e282, 1e283, 1e284, 1e285, 1e286,
1e287, 1e288, 1e289, 1e290, 1e291, 1e292, 1e293, 1e294, 1e295,
1e296, 1e297, 1e298, 1e299, 1e300, 1e301, 1e302, 1e303, 1e304,
1e305, 1e306, 1e307, 1e308};
bool powers_of_ten() {
std::cout << __func__ << std::endl;
char buf[1024];
const bool is_pow_correct{1e-308 == std::pow(10,-308)};
const int start_point = is_pow_correct ? -10000 : -307;
if(!is_pow_correct) {
std::cout << "On your system, the pow function is busted. Sorry about that. " << std::endl;
}
for (int i = start_point; i <= 308; ++i) {// large negative values should be zero.
const size_t n = std::snprintf(buf, sizeof(buf), "1e%d", i);
if (n >= sizeof(buf)) { std::abort(); }
std::fflush(nullptr);
const double expected = ((i >= -307) ? testing_power_of_ten[i + 307]: std::pow(10, i));
if(!test_ondemand<double>(padded_string(buf, n), [&](double actual) {
if(actual!=expected) {
std::cerr << "JSON '" << buf << " parsed to ";
std::fprintf( stderr," %18.18g instead of %18.18g\n", actual, expected); // formatting numbers is easier with printf
SIMDJSON_SHOW_DEFINE(FLT_EVAL_METHOD);
return false;
}
return true;
})) {
return false;
} // if
} // for i
std::printf("Powers of 10 can be parsed.\n");
return true;
}
void github_issue_1273() {
padded_string bad(std::string_view("0.0300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000024000000000000000000000000000000000000000000000000000000000000122978293824"));
simdjson::builtin::ondemand::parser parser;
simdjson_unused auto blah=parser.iterate(bad);
double x;
simdjson_unused auto blah2=blah.get(x);
}
bool old_crashes() {
github_issue_1273();
return true;
}
bool run() {
return small_integers() &&
powers_of_two() &&
powers_of_ten() &&
old_crashes();
}
} // namespace number_tests
int main(int argc, char *argv[]) {
return test_main(argc, argv, number_tests::run);
}
tests/ondemand/ondemand_ordering_tests.cpp
+154
View File
@@ -0,0 +1,154 @@
#include <cinttypes>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <string>
#include <vector>
#include <cmath>
#include <set>
#include <sstream>
#include <utility>
#include <unistd.h>
#include "simdjson.h"
#include "test_ondemand.h"
using namespace simdjson;
using namespace simdjson::builtin;
namespace ordering_tests {
using namespace std;
#if SIMDJSON_EXCEPTIONS
auto json = "{\"coordinates\":[{\"x\":1.1,\"y\":2.2,\"z\":3.3}]}"_padded;
bool in_order_object_index() {
TEST_START();
ondemand::parser parser{};
auto doc = parser.iterate(json);
double x{0};
double y{0};
double z{0};
for (ondemand::object point_object : doc["coordinates"]) {
x += double(point_object["x"]);
y += double(point_object["y"]);
z += double(point_object["z"]);
}
return (x == 1.1) && (y == 2.2) && (z == 3.3);
}
bool in_order_object_find_field_unordered() {
TEST_START();
ondemand::parser parser{};
auto doc = parser.iterate(json);
double x{0};
double y{0};
double z{0};
for (ondemand::object point_object : doc["coordinates"]) {
x += double(point_object.find_field_unordered("x"));
y += double(point_object.find_field_unordered("y"));
z += double(point_object.find_field_unordered("z"));
}
return (x == 1.1) && (y == 2.2) && (z == 3.3);
}
bool in_order_object_find_field() {
TEST_START();
ondemand::parser parser{};
auto doc = parser.iterate(json);
double x{0};
double y{0};
double z{0};
for (ondemand::object point_object : doc["coordinates"]) {
x += double(point_object.find_field("x"));
y += double(point_object.find_field("y"));
z += double(point_object.find_field("z"));
}
return (x == 1.1) && (y == 2.2) && (z == 3.3);
}
bool out_of_order_object_index() {
TEST_START();
ondemand::parser parser{};
auto doc = parser.iterate(json);
double x{0};
double y{0};
double z{0};
for (ondemand::object point_object : doc["coordinates"]) {
z += double(point_object["z"]);
x += double(point_object["x"]);
y += double(point_object["y"]);
}
return (x == 1.1) && (y == 2.2) && (z == 3.3);
}
bool out_of_order_object_find_field_unordered() {
TEST_START();
ondemand::parser parser{};
auto doc = parser.iterate(json);
double x{0};
double y{0};
double z{0};
for (ondemand::object point_object : doc["coordinates"]) {
z += double(point_object.find_field_unordered("z"));
x += double(point_object.find_field_unordered("x"));
y += double(point_object.find_field_unordered("y"));
}
return (x == 1.1) && (y == 2.2) && (z == 3.3);
}
bool out_of_order_object_find_field() {
TEST_START();
ondemand::parser parser{};
auto doc = parser.iterate(json);
double x{0};
double y{0};
double z{0};
for (ondemand::object point_object : doc["coordinates"]) {
z += double(point_object.find_field("z"));
ASSERT_ERROR( point_object.find_field("x"), NO_SUCH_FIELD );
ASSERT_ERROR( point_object.find_field("y"), NO_SUCH_FIELD );
}
return (x == 0) && (y == 0) && (z == 3.3);
}
bool foreach_object_field_lookup() {
TEST_START();
ondemand::parser parser{};
auto doc = parser.iterate(json);
double x{0};
double y{0};
double z{0};
for (ondemand::object point_object : doc["coordinates"]) {
for (auto field : point_object) {
if (field.key() == "z") { z += double(field.value()); }
else if (field.key() == "x") { x += double(field.value()); }
else if (field.key() == "y") { y += double(field.value()); }
}
}
return (x == 1.1) && (y == 2.2) && (z == 3.3);
}
#endif // SIMDJSON_EXCEPTIONS
bool run() {
return
#if SIMDJSON_EXCEPTIONS
in_order_object_index() &&
in_order_object_find_field_unordered() &&
in_order_object_find_field() &&
out_of_order_object_index() &&
out_of_order_object_find_field_unordered() &&
out_of_order_object_find_field() &&
foreach_object_field_lookup() &&
#endif // SIMDJSON_EXCEPTIONS
true;
}
} // namespace ordering_tests
int main(int argc, char *argv[]) {
return test_main(argc, argv, ordering_tests::run);
}
tests/ondemand/ondemand_parse_api_tests.cpp
+57
View File
@@ -0,0 +1,57 @@
#include <cinttypes>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <string>
#include <vector>
#include <cmath>
#include <set>
#include <sstream>
#include <utility>
#include <unistd.h>
#include "simdjson.h"
#include "test_ondemand.h"
using namespace simdjson;
using namespace simdjson::builtin;
namespace parse_api_tests {
using namespace std;
const padded_string BASIC_JSON = "[1,2,3]"_padded;
const padded_string BASIC_NDJSON = "[1,2,3]\n[4,5,6]"_padded;
const padded_string EMPTY_NDJSON = ""_padded;
bool parser_iterate() {
TEST_START();
ondemand::parser parser;
auto doc = parser.iterate(BASIC_JSON);
ASSERT_SUCCESS( doc.get_array() );
return true;
}
#if SIMDJSON_EXCEPTIONS
bool parser_iterate_exception() {
TEST_START();
ondemand::parser parser;
auto doc = parser.iterate(BASIC_JSON);
simdjson_unused ondemand::array array = doc;
return true;
}
#endif // SIMDJSON_EXCEPTIONS
bool run() {
return parser_iterate() &&
#if SIMDJSON_EXCEPTIONS
parser_iterate_exception() &&
#endif // SIMDJSON_EXCEPTIONS
true;
}
}
int main(int argc, char *argv[]) {
return test_main(argc, argv, parse_api_tests::run);
}
tests/ondemand/ondemand_twitter_tests.cpp
+210
View File
@@ -0,0 +1,210 @@
#include <cinttypes>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <string>
#include <vector>
#include <cmath>
#include <set>
#include <sstream>
#include <utility>
#include <unistd.h>
#include "simdjson.h"
#include "test_ondemand.h"
using namespace simdjson;
using namespace simdjson::builtin;
namespace twitter_tests {
using namespace std;
bool twitter_count() {
TEST_START();
padded_string json;
ASSERT_SUCCESS( padded_string::load(TWITTER_JSON).get(json) );
ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) {
uint64_t count;
ASSERT_SUCCESS( doc_result["search_metadata"]["count"].get(count) );
ASSERT_EQUAL( count, 100 );
return true;
}));
TEST_SUCCEED();
}
#if SIMDJSON_EXCEPTIONS
bool twitter_example() {
TEST_START();
padded_string json;
ASSERT_SUCCESS( padded_string::load(TWITTER_JSON).get(json) );
ondemand::parser parser;
auto doc = parser.iterate(json);
for (ondemand::object tweet : doc["statuses"]) {
uint64_t id = tweet["id"];
std::string_view text = tweet["text"];
std::string_view screen_name = tweet["user"]["screen_name"];
uint64_t retweets = tweet["retweet_count"];
uint64_t favorites = tweet["favorite_count"];
(void) id;
(void) text;
(void) retweets;
(void) favorites;
(void) screen_name;
}
TEST_SUCCEED();
}
#endif // SIMDJSON_EXCEPTIONS
bool twitter_default_profile() {
TEST_START();
padded_string json;
ASSERT_SUCCESS( padded_string::load(TWITTER_JSON).get(json) );
ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) {
// Print users with a default profile.
set<string_view> default_users;
for (auto tweet : doc_result["statuses"]) {
auto user = tweet["user"].get_object();
// We have to get the screen name before default_profile because it appears first
std::string_view screen_name;
ASSERT_SUCCESS( user["screen_name"].get(screen_name) );
bool default_profile;
ASSERT_SUCCESS( user["default_profile"].get(default_profile) );
if (default_profile) {
default_users.insert(screen_name);
}
}
ASSERT_EQUAL( default_users.size(), 86 );
return true;
}));
TEST_SUCCEED();
}
bool twitter_image_sizes() {
TEST_START();
padded_string json;
ASSERT_SUCCESS( padded_string::load(TWITTER_JSON).get(json) );
ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) {
// Print image names and sizes
set<pair<uint64_t, uint64_t>> image_sizes;
for (auto tweet : doc_result["statuses"]) {
auto media = tweet["entities"]["media"];
if (!media.error()) {
for (auto image : media) {
uint64_t id_val;
std::string_view id_string;
ASSERT_SUCCESS( image["id"].get(id_val) );
ASSERT_SUCCESS( image["id_str"].get(id_string) );
std::cout << "id = " << id_val << std::endl;
std::cout << "id_string = " << id_string << std::endl;
for (auto size : image["sizes"].get_object()) {
std::string_view size_key;
ASSERT_SUCCESS( size.unescaped_key().get(size_key) );
std::cout << "Type of image size = " << size_key << std::endl;
uint64_t width, height;
ASSERT_SUCCESS( size.value()["w"].get(width) );
ASSERT_SUCCESS( size.value()["h"].get(height) );
image_sizes.insert(make_pair(width, height));
}
}
}
}
ASSERT_EQUAL( image_sizes.size(), 15 );
return true;
}));
TEST_SUCCEED();
}
#if SIMDJSON_EXCEPTIONS
bool twitter_count_exception() {
TEST_START();
padded_string json;
ASSERT_SUCCESS( padded_string::load(TWITTER_JSON).get(json) );
ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) {
uint64_t count = doc_result["search_metadata"]["count"];
ASSERT_EQUAL( count, 100 );
return true;
}));
TEST_SUCCEED();
}
bool twitter_default_profile_exception() {
TEST_START();
padded_string json = padded_string::load(TWITTER_JSON);
ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) {
// Print users with a default profile.
set<string_view> default_users;
for (auto tweet : doc_result["statuses"]) {
ondemand::object user = tweet["user"];
// We have to get the screen name before default_profile because it appears first
std::string_view screen_name = user["screen_name"];
if (user["default_profile"]) {
default_users.insert(screen_name);
}
}
ASSERT_EQUAL( default_users.size(), 86 );
return true;
}));
TEST_SUCCEED();
}
/*
* Fun fact: id and id_str can differ:
* 505866668485386240 and 505866668485386241.
* Presumably, it is because doubles are used
* at some point in the process and the number
* 505866668485386241 cannot be represented as a double.
* (not our fault)
*/
bool twitter_image_sizes_exception() {
TEST_START();
padded_string json = padded_string::load(TWITTER_JSON);
ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) {
// Print image names and sizes
set<pair<uint64_t, uint64_t>> image_sizes;
for (auto tweet : doc_result["statuses"]) {
auto media = tweet["entities"]["media"];
if (!media.error()) {
for (auto image : media) {
std::cout << "id = " << uint64_t(image["id"]) << std::endl;
std::cout << "id_string = " << std::string_view(image["id_str"]) << std::endl;
for (auto size : image["sizes"].get_object()) {
std::cout << "Type of image size = " << std::string_view(size.unescaped_key()) << std::endl;
// NOTE: the uint64_t is required so that each value is actually parsed before the pair is created
image_sizes.insert(make_pair<uint64_t,uint64_t>(size.value()["w"], size.value()["h"]));
}
}
}
}
ASSERT_EQUAL( image_sizes.size(), 15 );
return true;
}));
TEST_SUCCEED();
}
#endif // SIMDJSON_EXCEPTIONS
bool run() {
return
twitter_count() &&
twitter_default_profile() &&
twitter_image_sizes() &&
#if SIMDJSON_EXCEPTIONS
twitter_count_exception() &&
twitter_example() &&
twitter_default_profile_exception() &&
twitter_image_sizes_exception() &&
#endif // SIMDJSON_EXCEPTIONS
true;
}
} // namespace twitter_tests
int main(int argc, char *argv[]) {
return test_main(argc, argv, twitter_tests::run);
}
tests/ondemand/test_ondemand.h
+59
View File
@@ -1,6 +1,7 @@
#ifndef ONDEMAND_TEST_ONDEMAND_H
#define ONDEMAND_TEST_ONDEMAND_H
#include <unistd.h>
#include "simdjson.h"
#include "cast_tester.h"
#include "test_macros.h"
@@ -28,4 +29,62 @@ bool test_ondemand_doc(const simdjson::padded_string &json, const F& f) {
return test_ondemand_doc(parser, json, f);
}
#define ONDEMAND_SUBTEST(NAME, JSON, TEST) \
{ \
std::cout << "- Subtest " << (NAME) << " - JSON: " << (JSON) << " ..." << std::endl; \
if (!test_ondemand_doc(JSON##_padded, [&](auto doc) { \
return (TEST); \
})) { \
return false; \
} \
}
const size_t AMAZON_CELLPHONES_NDJSON_DOC_COUNT = 793;
#define SIMDJSON_SHOW_DEFINE(x) printf("%s=%s\n", #x, STRINGIFY(x))
template<typename F>
int test_main(int argc, char *argv[], const F& test_function) {
std::cout << std::unitbuf;
int c;
while ((c = getopt(argc, argv, "a:")) != -1) {
switch (c) {
case 'a': {
const simdjson::implementation *impl = simdjson::available_implementations[optarg];
if (!impl) {
std::fprintf(stderr, "Unsupported architecture value -a %s\n", optarg);
return EXIT_FAILURE;
}
simdjson::active_implementation = impl;
break;
}
default:
std::fprintf(stderr, "Unexpected argument %c\n", c);
return EXIT_FAILURE;
}
}
// this is put here deliberately to check that the documentation is correct (README),
// should this fail to compile, you should update the documentation:
if (simdjson::active_implementation->name() == "unsupported") {
std::printf("unsupported CPU\n");
std::abort();
}
// We want to know what we are testing.
// Next line would be the runtime dispatched implementation but that's not necessarily what gets tested.
// std::cout << "Running tests against this implementation: " << simdjson::active_implementation->name();
// Rather, we want to display builtin_implementation()->name().
// In practice, by default, we often end up testing against fallback.
std::cout << "builtin_implementation -- " << simdjson::builtin_implementation()->name() << std::endl;
std::cout << "------------------------------------------------------------" << std::endl;
std::cout << "Running tests." << std::endl;
if (test_function()) {
std::cout << "Success!" << std::endl;
return EXIT_SUCCESS;
} else {
std::cerr << "FAILED." << std::endl;
return EXIT_FAILURE;
}
}
#endif // ONDEMAND_TEST_ONDEMAND_H
tests/test_macros.h
-1
View File
@@ -101,5 +101,4 @@ simdjson_really_inline bool assert_true(bool value, const char *operation = "res
#define TEST_FAIL(MESSAGE) do { std::cerr << "FAIL: " << (MESSAGE) << std::endl; return false; } while (0);
#define TEST_SUCCEED() do { return true; } while (0);
#endif // TEST_MACROS_H