Compare commits

..

88 Commits

Author SHA1 Message Date
Daniel Lemire 450cb7b0f8 Merge branch 'dlemire/nopadding' of github.com:simdjson/simdjson into dlemire/nopadding 2021-07-23 23:40:41 -04:00
Daniel Lemire 2907912b1d Guarding the peek/advance. 2021-07-23 23:40:26 -04:00
Daniel Lemire b6aff44c42 Syncing the singleheader files. 2021-07-23 23:12:03 -04:00
Daniel Lemire ce11f41840 Merge branch 'dlemire/nopadding' of github.com:simdjson/simdjson into dlemire/nopadding 2021-07-23 23:06:32 -04:00
Daniel Lemire 7c8bc650a7 Turning on developer mode explicitly throughout. 2021-07-23 23:06:08 -04:00
Daniel Lemire 657489e387 More fixes. 2021-07-23 22:27:46 -04:00
Daniel Lemire 60b6a16edc Let us stress the memory overflow in more tests. 2021-07-23 21:56:29 -04:00
Daniel Lemire 2ceca3e510 More tweaking to fix problems exposed by the fuzzers. 2021-07-23 21:36:40 -04:00
Daniel Lemire d8b6a8c9d3 Merge branch 'dlemire/nopadding' of github.com:simdjson/simdjson into dlemire/nopadding 2021-07-23 20:35:20 -04:00
Daniel Lemire bbc9bd059a Some tweaking. 2021-07-23 20:35:12 -04:00
Daniel Lemire 68df233b4b Removing a couple of tests from circle ci. 2021-07-23 18:07:43 -04:00
Daniel Lemire dc66d6133a Fixing comment. 2021-07-23 17:48:21 -04:00
Daniel Lemire 13055a4601 Using an alternative instrinsic. 2021-07-23 17:47:05 -04:00
Daniel Lemire a7ff410532 Various fixes for the number parsing in the DOM front-end. Basically, we had
a mess whereas a SWAR optimization processing 8 digits would sometimes be called,
despite the fact that it would be unsafe and/or it would not be called, and we would
end up leaving performance on the floor.
2021-07-23 17:39:09 -04:00
Daniel Lemire d72cba29be Minor tweaks. 2021-07-23 16:25:08 -04:00
Daniel Lemire 757db1b2f0 Removing padding from DOM API. 2021-07-23 15:18:49 -04:00
Daniel Lemire 7175731e09 Removing the special-case 2021-07-23 14:07:32 -04:00
Daniel Lemire f6fc786b05 Disabling some change made to the DOM kernel which was evidently made to allow padding-less parsing. 2021-07-23 14:01:10 -04:00
Daniel Lemire 8ce1e52e4f Merge branch 'master' into dlemire/nopadding 2021-07-23 11:43:39 -04:00
Daniel Lemire b12014f9e4 Let us try some setups in debug mode (for faster builds). 2021-07-23 11:20:51 -04:00
Daniel Lemire 1a433fc5b4 Enabling a broad range of tests. 2021-07-23 10:51:43 -04:00
Daniel Lemire 21845f1f6a Documenting iterate_many + removing remnants of padding in On Demand. 2021-07-23 10:37:08 -04:00
Daniel Lemire dc86daa4ec Removing the padding. 2021-07-22 17:32:06 -04:00
Daniel Lemire 910686ffa8 Merge branch 'dlemire/nopadding' of github.com:simdjson/simdjson into dlemire/nopadding 2021-07-22 15:41:01 -04:00
Daniel Lemire b1a3943a8b We should now be able to iterate over a string_view. 2021-07-22 15:40:45 -04:00
Daniel Lemire 70a0d62f51 Merge branch 'dlemire/backport_jkeiser_fix_for_issue1632' into dlemire/nopadding 2021-07-22 15:27:41 -04:00
Daniel Lemire bb0ed22ab8 Removing some of the padding in the tests. 2021-07-22 15:26:41 -04:00
Daniel Lemire d7770e9a6b Disabling 'json[N-1]' test. 2021-07-22 14:24:39 -04:00
Daniel Lemire 0e8b68683d Merge branch 'dlemire/backport_jkeiser_fix_for_issue1632' into dlemire/nopadding 2021-07-22 14:03:23 -04:00
Daniel Lemire 1fe50cb777 Trimming. 2021-07-22 14:02:30 -04:00
Daniel Lemire f2d419dda6 Basic guarding. 2021-07-22 13:42:49 -04:00
Daniel Lemire 57ca29e151 Merge branch 'dlemire/backport_jkeiser_fix_for_issue1632' into dlemire/nopadding 2021-07-22 13:23:08 -04:00
Daniel Lemire 24068a089e Merge branch 'master' into dlemire/backport_jkeiser_fix_for_issue1632 2021-07-22 12:56:18 -04:00
Daniel Lemire e4bba2df9d Do not needlessly do perf checks. 2021-07-22 10:52:05 -04:00
Daniel Lemire ca9391c5a9 Let us run more CI tests. 2021-07-22 10:44:10 -04:00
Daniel Lemire 91b8fa4c4b Fixing a CI issue. 2021-07-22 10:38:59 -04:00
Daniel Lemire 081c3ca7c8 Merged. 2021-07-22 10:17:37 -04:00
Daniel Lemire bbeec66a93 Merge branch 'jkeiser/no-padding-scalar' into dlemire/nopadding 2021-07-21 13:45:10 -04:00
Daniel Lemire 7dcd43e82a Merge branch 'jkeiser/no-padding' into dlemire/nopadding 2021-07-21 11:19:43 -04:00
Daniel Lemire c3f7eff019 Adding some commentary 2021-07-19 17:08:51 -04:00
Daniel Lemire d0e2a7fb00 Minor fixes 2021-07-19 16:53:11 -04:00
Daniel Lemire 9cace04da3 Adding more tests. 2021-07-19 16:33:58 -04:00
Daniel Lemire 2e688a2825 Removing two other asserts. 2021-07-19 16:27:29 -04:00
Daniel Lemire 4501aebee8 Disabling a couple of asserts. 2021-07-19 16:23:48 -04:00
Daniel Lemire df24da314f Cleaning. 2021-07-19 15:58:17 -04:00
Daniel Lemire 8b2bff981f Messy temporary work. 2021-07-19 15:34:48 -04:00
Daniel Lemire a34996939c Adding a duplicate of the existing test. 2021-07-19 13:55:10 -04:00
Daniel Lemire e8e731a311 Merge branch 'master' into dlemire/backport_jkeiser_fix_for_issue1632 2021-07-19 12:17:17 -04:00
Daniel Lemire f364e354e2 Minor fixes. 2021-07-19 11:54:46 -04:00
Daniel Lemire 3c5c2d2404 This backports (isolate) a fix by jkeiser of issue 1632. 2021-07-19 11:13:37 -04:00
Daniel Lemire 3300d8463b Adding type test. 2021-07-18 20:53:57 -04:00
Daniel Lemire 267e70d550 Adding another case. 2021-07-18 16:53:35 -04:00
Daniel Lemire 4b118bd9e1 Verifying issue. 2021-07-18 13:00:46 -04:00
Daniel Lemire e5d0e92116 Minor correction. 2021-06-23 13:36:41 -06:00
Daniel Lemire b3b108b120 Adding the functions that I meant to add earlier. 2021-06-23 13:36:41 -06:00
Daniel Lemire 40f522ab2d Adding potentially useless definition. 2021-06-23 13:36:41 -06:00
Daniel Lemire 5a6894cd3e This adds "guarded" (i.e., memory-safe) number parsing routines. 2021-06-23 13:36:41 -06:00
John Keiser 90409897eb Add __SIMDJSON_CHECK_EOF feature flag 2021-06-23 13:36:41 -06:00
John Keiser 78d1cd96a9 Split ondemand_object_tests into smaller files 2021-06-23 13:36:41 -06:00
John Keiser 6a1f8fc5c3 Don't step off the end of the index buffer 2021-06-23 13:36:37 -06:00
John Keiser 42a31ad096 Fix ARM compile errors 2021-06-23 11:52:00 -06:00
John Keiser ac647e6c11 Don't change the order of fields to avoid perf changes 2021-06-21 10:46:01 -06:00
John Keiser 45fbd8c4c5 Make len available to any dom_parser_implementation user 2021-06-21 10:38:56 -06:00
John Keiser 6ba9ad3ed9 Remove need for padding from SIMD string parsing 2021-06-19 21:20:31 -07:00
John Keiser bab51e5f08 Remove unlikely from null / bool parsing 2021-06-19 10:35:05 -07:00
Daniel Lemire f45c59f27c I suspect that sanitizers might increase stack usage, to combat that you may set SIMDJSON_NO_FORCE_INLINING (#1614) 2021-06-08 10:58:20 -04:00
Daniel Lemire ba0406f021 Minor changes to compilation test. 2021-06-07 10:21:39 -04:00
John Keiser b02aedb5bb Don't use padding parsing booleans 2021-06-06 16:56:28 -07:00
John Keiser da4de2e0b8 Don't read padding at all for root booleans and numbers 2021-06-06 15:04:08 -07:00
John Keiser eda0f190ea Don't allocate when parsing root number 2021-06-06 13:01:46 -07:00
John Keiser 633784a4a2 Merge issue 2021-06-06 11:51:18 -07:00
Daniel Lemire 10a7b9bd25 Merge branch 'master' into jkeiser/no-padding 2021-06-04 17:15:00 -04:00
Daniel Lemire 0d8f6f5827 Merging again. Fixing issue. 2021-05-27 20:43:51 -04:00
Daniel Lemire 0c14707b78 Merge branch 'master' into jkeiser/no-padding 2021-05-27 20:42:36 -04:00
Daniel Lemire c0fb5f7474 Another merge issue. 2021-05-27 20:30:25 -04:00
Daniel Lemire 930b0beb6d Missing #ifdef SIMDJSON_DEVELOPMENT_CHECKS
that got destroyed in the merge. Putting it back.
2021-05-27 18:27:20 -04:00
Daniel Lemire dadfb519cd Minor correction. 2021-05-27 18:04:01 -04:00
Daniel Lemire 595c628965 Merge branch 'master' into jkeiser/no-padding 2021-05-27 17:53:43 -04:00
Daniel Lemire 76f7e4ae04 Merge branch 'master' into jkeiser/no-padding 2021-05-07 23:03:39 -04:00
Daniel Lemire d7b08c84c7 Merge branch 'master' into jkeiser/no-padding 2021-04-23 10:18:42 -04:00
John Keiser 4d0f1021ff Merge pull request #1516 from simdjson/dlemire/guarded_number_parsing
guarded number parsing (to be merged into Remove padding access from iteration #1511)
2021-03-31 11:18:18 -07:00
Daniel Lemire 42c854124f Adding the functions that I meant to add earlier. 2021-03-23 16:39:53 -04:00
Daniel Lemire 081f954465 Adding potentially useless definition. 2021-03-23 15:20:25 -04:00
Daniel Lemire 1fbd6047c0 This adds "guarded" (i.e., memory-safe) number parsing routines. 2021-03-23 12:00:39 -04:00
John Keiser 09601eb480 Add __SIMDJSON_CHECK_EOF feature flag 2021-03-22 11:03:00 -07:00
John Keiser 02762fa352 Split ondemand_object_tests into smaller files 2021-03-21 14:53:16 -07:00
John Keiser a334303119 Don't step off the end of the index buffer 2021-03-21 14:22:00 -07:00
John Keiser 7f545a8996 Normalize the words "position" and "end" 2021-03-20 14:23:38 -07:00
314 changed files with 11334 additions and 28036 deletions
+10 -2
View File
@@ -208,6 +208,14 @@ jobs:
executor: clang10
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
steps: [ cmake_test ]
nocheckeof-clang10:
description: Validate that when __SIMDJSON_CHECK_EOF=0, everything still succeeds
environment:
CXXFLAGS: -D__SIMDJSON_CHECK_EOF=0
CMAKE_BUILD_FLAGS: --target ondemand_tests
CTEST_FLAGS: --output-on-failure -R ondemand_
executor: clang10
steps: [ cmake_test ]
# dynamic
dynamic-gcc10:
description: Build and run tests on GCC 10 and AVX 2 with a cmake dynamic build
@@ -284,7 +292,7 @@ workflows:
- libcpp-clang10
# full single-implementation tests
- sanitize-gcc10
# - sanitize-gcc10 # gcc10 + sanitize uses a lot of memory?
- sanitize-clang10
- threadsanitize-gcc10
- threadsanitize-clang10
@@ -303,7 +311,7 @@ workflows:
# sanitized single-implementation tests
- sanitize-haswell-gcc10
# - sanitize-haswell-gcc10 # gcc10 + sanitize uses a lot of memory?
- sanitize-haswell-clang10
# testing "just the library"
+2 -2
View File
@@ -19,8 +19,8 @@ task:
build_script:
- mkdir build
- cd build
- cmake -DSIMDJSON_BASH=OFF -DSIMDJSON_GIT=OFF ..
- make
- cmake -DSIMDJSON_BASH=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_GIT=OFF ..
- cmake --build .
test_script:
- cd build
- ctest --output-on-failure -LE explicitonly
+59 -21
View File
@@ -1,4 +1,46 @@
kind: pipeline
name: i386-gcc # we do not support 32-bit systems, but we run tests
platform: { os: linux, arch: amd64 }
steps:
- name: Build and Test
image: i386/ubuntu
environment:
CC: gcc
CXX: g++
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_DEVELOPER_MODE=ON
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
- apt-get install -y g++ cmake gcc git
- mkdir build
- cd build
- cmake $CMAKE_FLAGS ..
- cmake --build . $BUILD_FLAGS
- ctest $CTEST_FLAGS
---
kind: pipeline
name: i386-clang # we do not support 32-bit systems, but we run tests
platform: { os: linux, arch: amd64 }
steps:
- name: Build and Test
image: i386/ubuntu
environment:
CC: clang-6.0
CXX: clang++-6.0
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_DEVELOPER_MODE=ON
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
- apt-get install -y clang++-6.0 cmake git
- mkdir build
- cd build
- cmake $CMAKE_FLAGS ..
- cmake --build . $BUILD_FLAGS
- ctest $CTEST_FLAGS
---
kind: pipeline
name: gcc9
platform: { os: linux, arch: amd64 }
steps:
@@ -8,7 +50,7 @@ steps:
CC: gcc
CXX: g++
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -19,7 +61,6 @@ steps:
- cmake $CMAKE_FLAGS ..
- cmake --build . $BUILD_FLAGS
- ctest $CTEST_FLAGS -L acceptance -LE per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=icelake ctest $CTEST_FLAGS -L per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=haswell ctest $CTEST_FLAGS -L per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=westmere ctest $CTEST_FLAGS -L per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation
@@ -36,7 +77,7 @@ steps:
CC: clang-6.0
CXX: clang++-6.0
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- mkdir build
@@ -44,7 +85,6 @@ steps:
- cmake $CMAKE_FLAGS ..
- cmake --build . $BUILD_FLAGS
- ctest $CTEST_FLAGS -L acceptance -LE per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=icelake ctest $CTEST_FLAGS -L per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=haswell ctest $CTEST_FLAGS -L per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=westmere ctest $CTEST_FLAGS -L per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation
@@ -60,7 +100,7 @@ steps:
CC: gcc
CXX: g++
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -82,7 +122,7 @@ steps:
environment:
CC: clang-9
CXX: clang++-9
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON
BUILD_FLAGS: -- -j
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
@@ -102,7 +142,7 @@ steps:
CC: gcc
CXX: g++
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -113,7 +153,6 @@ steps:
- cmake $CMAKE_FLAGS ..
- cmake --build . $BUILD_FLAGS
- ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L acceptance -LE per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=icelake ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=haswell ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=westmere ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=fallback ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
@@ -129,7 +168,7 @@ steps:
environment:
CC: clang-9
CXX: clang++-9
CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
BUILD_FLAGS: -- -j
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
@@ -138,7 +177,6 @@ steps:
- cmake $CMAKE_FLAGS ..
- cmake --build . $BUILD_FLAGS
- ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L acceptance -LE per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=icelake ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=haswell ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=westmere ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=fallback ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
@@ -154,7 +192,7 @@ steps:
environment:
CC: clang-11
CXX: clang++-11
CMAKE_FLAGS: -GNinja
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -GNinja
BUILD_FLAGS:
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
CXXFLAGS: -std=c++20 -stdlib=libc++
@@ -175,7 +213,7 @@ steps:
CC: gcc
CXX: g++
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -199,7 +237,7 @@ steps:
environment:
CC: clang-6.0
CXX: clang++-6.0
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON
BUILD_FLAGS: -- -j
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
@@ -222,7 +260,7 @@ steps:
CC: gcc
CXX: g++
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -243,7 +281,7 @@ steps:
environment:
CC: clang-6.0
CXX: clang++-6.0
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON
BUILD_FLAGS: -- -j
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
@@ -264,7 +302,7 @@ steps:
image: gcc:8
environment:
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
CC: gcc
CXX: g++
@@ -291,7 +329,7 @@ steps:
environment:
CC: clang-6.0
CXX: clang++-6.0
CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=arm64;fallback
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=arm64;fallback
BUILD_FLAGS: -- -j
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
@@ -318,7 +356,7 @@ steps:
CC: clang-9
CXX: clang++-9
BUILD_FLAGS: -- -j 4
CMAKE_FLAGS: -GNinja -DBUILD_SHARED_LIBS=OFF
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -GNinja -DBUILD_SHARED_LIBS=OFF
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
CXXFLAGS: -stdlib=libc++
commands:
@@ -339,7 +377,7 @@ steps:
CC: clang-9
CXX: clang++-9
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
CXXFLAGS: -stdlib=libc++
commands:
@@ -360,7 +398,7 @@ steps:
CC: clang-7
CXX: clang++-7
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
CXXFLAGS: -stdlib=libc++
commands:
@@ -380,7 +418,7 @@ steps:
CC: gcc
CXX: g++
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DSIMDJSON_EXCEPTIONS=OFF
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_EXCEPTIONS=OFF
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
-12
View File
@@ -1,12 +0,0 @@
# https://editorconfig.org/
root = true
# Conservatively avoid changing defaults for other file types, e.g. raw json files for test cases,
# Makefiles, etc.
[*.{cpp,h,md}]
charset = utf-8
end_of_line = lf
indent_size = 2
indent_style = space
insert_final_newline = true
tab_width = 2
trim_trailing_whitespace = true
-1
View File
@@ -78,7 +78,6 @@
.gitattributes export-ignore
.gitignore export-ignore
.editorconfig export-ignore
# Sources
*.c text eol=lf diff=c
+2 -2
View File
@@ -9,8 +9,8 @@ jobs:
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-24
View File
@@ -1,24 +0,0 @@
name: CIFuzz
on: [pull_request]
jobs:
Fuzzing:
runs-on: ubuntu-latest
steps:
- name: Build Fuzzers
id: build
uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
with:
oss-fuzz-project-name: 'simdjson'
dry-run: false
- name: Run Fuzzers
uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
with:
oss-fuzz-project-name: 'simdjson'
fuzz-seconds: 600
dry-run: false
- name: Upload Crash
uses: actions/upload-artifact@v1
if: failure() && steps.build.outcome == 'success'
with:
name: artifacts
path: ./out/artifacts
-33
View File
@@ -1,33 +0,0 @@
name: Debian
on: [push, pull_request]
defaults:
run:
shell: sh
permissions:
contents: read
jobs:
pkg-config:
runs-on: ubuntu-latest
container:
image: debian:testing
steps:
- uses: actions/checkout@v3
- name: Install dependencies
run: |
apt -y update
apt -y --no-install-recommends install g++ cmake make pkg-config
- name: Build and install
run: |
cmake -B build
cmake --build build
cmake --install build
- name: Test pkg-config
run: g++ examples/quickstart/quickstart.cpp $(pkg-config --cflags --libs simdjson)
@@ -6,7 +6,7 @@ jobs:
whitespace:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v2
- name: Remove whitespace and check the diff
run: |
set -eu
+3 -3
View File
@@ -37,14 +37,14 @@ jobs:
chmod +x llvm.sh
sudo ./llvm.sh $CLANGVERSION
- uses: actions/checkout@v3
- uses: actions/checkout@v1
- uses: actions/cache@v3
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
- uses: actions/cache@v3
- uses: actions/cache@v2
id: cache-corpus
with:
path: out/
-33
View File
@@ -1,33 +0,0 @@
name: Macos
on: [push, pull_request]
jobs:
macos-build:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
- name: Use cmake
run: |
mkdir builddebug &&
cd builddebug &&
cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. &&
cmake --build . &&
ctest -j --output-on-failure -LE explicitonly &&
cd .. &&
mkdir build &&
cd build &&
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake --build . &&
ctest -j --output-on-failure -LE explicitonly &&
cmake --install . &&
echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
cd ../tests/installation_tests/find &&
mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. && cmake --build .
+63
View File
@@ -0,0 +1,63 @@
name: MinGW32-CI
on: [push, pull_request]
# Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.
# It is important to build static libraries because cmake is not smart enough under Windows/mingw to take care of the path. So
# with a dynamic library, you could get failures due to the fact that the EXE can't find its DLL.
jobs:
ci:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
name: windows-gcc
runs-on: windows-2016
env:
CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DBUILD_SHARED_LIBS=OFF .. ' if using the command line
CC: gcc
CXX: g++
steps: # To reproduce what is below, start a powershell with administrative rights, using scoop *is* a good idea
- uses: actions/checkout@v2
- uses: actions/cache@v2 # we cache the scoop setup with 32-bit GCC
id: cache
with:
path: |
C:\ProgramData\scoop
key: scoop32 # static key: should be good forever
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
- name: Setup Windows # This should almost never run if the cache works.
if: steps.cache.outputs.cache-hit != 'true'
shell: powershell
run: |
Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')
scoop install sudo --global
sudo scoop install git --global
sudo scoop install ninja --global
sudo scoop install cmake --global
sudo scoop install gcc --arch 32bit --global
$env:path
Write-Host 'Everything has been installed, you are good!'
- name: Build and Test 32-bit x86
shell: powershell
run: |
$ENV:PATH="C:\ProgramData\scoop\shims;C:\ProgramData\scoop\apps\gcc\current\bin;C:\ProgramData\scoop\apps\ninja\current;$ENV:PATH"
g++ --version
cmake --version
ninja --version
git --version
mkdir build32
cd build32
cmake -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
cmake --build . --target acceptance_tests --verbose
ctest -L acceptance --output-on-failure
+69
View File
@@ -0,0 +1,69 @@
name: MinGW64-CI
on: [push, pull_request]
# Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.
# It is important to build static libraries because cmake is not smart enough under Windows/mingw to take care of the path. So
# with a dynamic library, you could get failures due to the fact that the EXE can't find its DLL.
jobs:
ci:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
name: windows-gcc
runs-on: windows-2016
env:
CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DBUILD_SHARED_LIBS=OFF .. ' if using the command line
CC: gcc
CXX: g++
steps: # To reproduce what is below, start a powershell with administrative rights, using scoop *is* a good idea
- uses: actions/checkout@v2
- uses: actions/cache@v2 # we cache the scoop setup with 64-bit GCC
id: cache
with:
path: |
C:\ProgramData\scoop
key: scoop64 # static key: should be good forever
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
- name: Setup Windows # This should almost never run if the cache works.
if: steps.cache.outputs.cache-hit != 'true'
shell: powershell
run: |
Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')
scoop install sudo --global
sudo scoop install git --global
sudo scoop install ninja --global
sudo scoop install cmake --global
sudo scoop install gcc --arch 64bit --global
$env:path
Write-Host 'Everything has been installed, you are good!'
- name: Build and Test 64-bit x64
shell: powershell
run: |
$ENV:PATH="C:\ProgramData\scoop\shims;C:\ProgramData\scoop\apps\gcc\current\bin;C:\ProgramData\scoop\apps\ninja\current;$ENV:PATH"
g++ --version
cmake --version
ninja --version
git --version
mkdir build64
cd build64
cmake -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
cmake --build . --target acceptance_tests --verbose
ctest -L acceptance --output-on-failure
cd ..
mkdir build64debug
cd build64debug
cmake -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
cmake --build . --target acceptance_tests --verbose
ctest -L acceptance --output-on-failure
+9 -3
View File
@@ -17,15 +17,21 @@ jobs:
- msystem: "MINGW64"
install: mingw-w64-x86_64-libxml2 mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-clang
type: Release
- msystem: "MINGW32"
install: mingw-w64-i686-libxml2 mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-clang
type: Release
- msystem: "MINGW64"
install: mingw-w64-x86_64-libxml2 mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-clang
type: Debug
- msystem: "MINGW32"
install: mingw-w64-i686-libxml2 mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-clang
type: Debug
env:
CMAKE_GENERATOR: Ninja
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
@@ -38,6 +44,6 @@ jobs:
run: |
mkdir build
cd build
cmake -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON ..
cmake -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON ..
cmake --build . --verbose
ctest -j4 --output-on-failure -LE explicitonly
+8 -2
View File
@@ -19,15 +19,21 @@ jobs:
- msystem: "MINGW64"
install: mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-gcc
type: Release
- msystem: "MINGW32"
install: mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-gcc
type: Release
- msystem: "MINGW64"
install: mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-gcc
type: Debug
- msystem: "MINGW32"
install: mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-gcc
type: Debug
env:
CMAKE_GENERATOR: Ninja
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+2 -2
View File
@@ -15,8 +15,8 @@ jobs:
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-31
View File
@@ -1,31 +0,0 @@
name: Ubuntu 18.04 CI (LLVM 7)
on: [push, pull_request]
jobs:
ubuntu-build:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-18.04
env:
CC: clang-7
CXX: clang++-7
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
- name: install clang 7
run: |
sudo apt update
sudo apt install clang-7
- name: Use cmake
run: |
mkdir build &&
cd build &&
cmake -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake --build . &&
ctest -j --output-on-failure -LE explicitonly
+2 -2
View File
@@ -10,8 +10,8 @@ jobs:
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+2 -2
View File
@@ -9,8 +9,8 @@ jobs:
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+2 -2
View File
@@ -15,8 +15,8 @@ jobs:
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-38
View File
@@ -1,38 +0,0 @@
name: Ubuntu 20.04 CI (GCC 8)
on: [push, pull_request]
jobs:
ubuntu-build:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-20.04
env:
CXX: g++-8
CC: gcc-8
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
- name: Install GCC 8
run: sudo apt-get install -y g++-8
- name: Use cmake
run: |
mkdir builddebug &&
cd builddebug &&
cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. &&
cmake --build . &&
ctest -j --output-on-failure -LE explicitonly &&
cd .. &&
mkdir build &&
cd build &&
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake --build . &&
ctest -j --output-on-failure -LE explicitonly &&
cmake --install . &&
echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
cd ../tests/installation_tests/find &&
mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. && cmake --build .
+2 -2
View File
@@ -9,8 +9,8 @@ jobs:
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+2 -2
View File
@@ -9,8 +9,8 @@ jobs:
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+2 -2
View File
@@ -9,8 +9,8 @@ jobs:
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+2 -2
View File
@@ -9,8 +9,8 @@ jobs:
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+2 -2
View File
@@ -9,8 +9,8 @@ jobs:
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
-25
View File
@@ -1,25 +0,0 @@
name: Ubuntu 22.04 CI (CLANG 13)
on: [push, pull_request]
jobs:
ubuntu-build:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
- name: Install clang++-13
run: sudo apt-get install -y clang++-13
- name: Use cmake
run: |
mkdir build &&
cd build &&
CXX=clang++-13 cmake -DSIMDJSON_DEVELOPER_MODE=ON .. &&
cmake --build . &&
ctest -j --output-on-failure -LE explicitonly
-25
View File
@@ -1,25 +0,0 @@
name: Ubuntu 22.04 CI (GCC 12)
on: [push, pull_request]
jobs:
ubuntu-build:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
- name: Install gcc12
run: sudo apt-get install -y g++-12
- name: Use cmake
run: |
mkdir build &&
cd build &&
CXX=g++-12 cmake -DSIMDJSON_DEVELOPER_MODE=ON .. &&
cmake --build . &&
ctest -j --output-on-failure -LE explicitonly
-33
View File
@@ -1,33 +0,0 @@
name: Ubuntu 22.04 CI (GCC 11)
on: [push, pull_request]
jobs:
ubuntu-build:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
- name: Use cmake
run: |
mkdir builddebug &&
cd builddebug &&
cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. &&
cmake --build . &&
ctest -j --output-on-failure -LE explicitonly &&
cd .. &&
mkdir build &&
cd build &&
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake --build . &&
ctest -j --output-on-failure -LE explicitonly &&
cmake --install . &&
echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
cd ../tests/installation_tests/find &&
mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. && cmake --build .
@@ -1,4 +1,4 @@
name: VS17-CLANG-CI
name: VS15-CI
on: [push, pull_request]
@@ -7,19 +7,22 @@ jobs:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
name: windows-vs17
runs-on: windows-latest
name: windows-vs15
runs-on: windows-2016
strategy:
fail-fast: false
matrix:
include:
- {gen: Visual Studio 17 2022, arch: x64}
- {gen: Visual Studio 15 2017, arch: Win32, static: ON}
- {gen: Visual Studio 15 2017, arch: Win32, static: OFF}
- {gen: Visual Studio 15 2017, arch: x64, static: ON}
- {gen: Visual Studio 15 2017, arch: x64, static: OFF}
steps:
- name: checkout
uses: actions/checkout@v3
uses: actions/checkout@v2
- name: Configure
run: |
cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -T ClangCL -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build
cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -B build
- name: Build Debug
run: cmake --build build --config Debug --verbose
- name: Build Release
@@ -1,10 +1,10 @@
name: VS17-ARM-CI
name: VS16-ARM-CI
on: [push, pull_request]
jobs:
ci:
name: windows-vs17
name: windows-vs16
runs-on: windows-latest
strategy:
fail-fast: false
@@ -14,7 +14,7 @@ jobs:
- {arch: ARM64}
steps:
- name: checkout
uses: actions/checkout@v3
uses: actions/checkout@v2
- name: Use cmake
run: |
cmake -A ${{ matrix.arch }} -DCMAKE_CROSSCOMPILING=1 -DSIMDJSON_DEVELOPER_MODE=ON -D SIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_EXCEPTIONS=OFF -B build &&
@@ -1,4 +1,4 @@
name: VS17-CI
name: VS16-CI
on: [push, pull_request]
@@ -7,22 +7,22 @@ jobs:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
name: windows-vs17
name: windows-vs16
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
include:
- {gen: Visual Studio 17 2022, arch: Win32, shared: ON}
- {gen: Visual Studio 17 2022, arch: Win32, shared: OFF}
- {gen: Visual Studio 17 2022, arch: x64, shared: ON}
- {gen: Visual Studio 17 2022, arch: x64, shared: OFF}
- {gen: Visual Studio 16 2019, arch: Win32, static: ON}
- {gen: Visual Studio 16 2019, arch: Win32, static: OFF}
- {gen: Visual Studio 16 2019, arch: x64, static: ON}
- {gen: Visual Studio 16 2019, arch: x64, static: OFF}
steps:
- name: checkout
uses: actions/checkout@v3
uses: actions/checkout@v2
- name: Configure
run: |
cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build
cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -B build
- name: Build Debug
run: cmake --build build --config Debug --verbose
- name: Build Release
+31
View File
@@ -0,0 +1,31 @@
name: VS16-CLANG-CI
on: [push, pull_request]
jobs:
ci:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
name: windows-vs16
runs-on: windows-latest
steps:
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
- name: 'Run CMake with VS16 Clang'
uses: lukka/run-cmake@v3
with:
cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
buildDirectory: "${{ github.workspace }}/../../_temp/windows"
cmakeBuildType: Release
buildWithCMake: true
cmakeAppendedArgs: -T ClangCL -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF
buildWithCMakeArgs: --config Release
- name: 'Run CTest'
run: ctest -C Release -LE explicitonly --output-on-failure
working-directory: "${{ github.workspace }}/../../_temp/windows"
+45
View File
@@ -0,0 +1,45 @@
name: VS16-Ninja-CI
on: [push, pull_request]
jobs:
ci:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
name: windows-vs16
runs-on: windows-latest
steps:
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
- name: 'Run CMake with VS16'
uses: lukka/run-cmake@v2
with:
cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
buildDirectory: "${{ github.workspace }}/../../_temp/windows"
cmakeBuildType: Release
buildWithCMake: true
cmakeAppendedArgs: -G Ninja -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF
buildWithCMakeArgs: --config Release
- name: 'Run CTest'
run: ctest -C Release -LE explicitonly --output-on-failure
working-directory: "${{ github.workspace }}/../../_temp/windows"
- name: 'Install with CMake'
uses: lukka/run-cmake@v3
with:
cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
buildWithCMakeArgs: '--target install'
- name: 'Test Installation with CMake'
uses: lukka/run-cmake@v3
with:
cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
cmakeListsTxtPath: '${{ github.workspace }}/tests/installation_tests/find/CMakeLists.txt'
cmakeBuildType: Release
buildWithCMake: true
buildDirectory: '${{ github.workspace }}/tests/installation_tests/find/buildDirectory'
cmakeAppendedArgs: -G Ninja
buildWithCMakeArgs: '--config Release --verbose'
@@ -1,18 +1,18 @@
name: VS17-NoExcept-CI
name: VS16-NoExcept-CI
on: [push, pull_request]
jobs:
ci:
name: windows-vs17
name: windows-vs16
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
- name: 'Run CMake with VS17'
- name: 'Run CMake with VS16'
uses: lukka/run-cmake@v3
with:
cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
+2 -2
View File
@@ -172,7 +172,7 @@ install:
- export CMAKE_C_FLAGS="${CMAKE_CXX_FLAGS}"
- export CMAKE_FLAGS="-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} -DSIMDJSON_IMPLEMENTATION=ppc64;fallback";
- if [[ "${SANITIZE}" == "on" ]]; then
export CMAKE_FLAGS="${CMAKE_FLAGS} -DSIMDJSON_SANITIZE=ON";
export CMAKE_FLAGS="${CMAKE_FLAGS} -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON";
export ASAN_OPTIONS="detect_leaks=0";
fi
- if [[ "${STATIC}" == "on" ]]; then
@@ -183,7 +183,7 @@ install:
script:
- mkdir build
- cd build
- cmake $CMAKE_FLAGS ..
- cmake -DSIMDJSON_DEVELOPER_MODE=ON $CMAKE_FLAGS ..
- cmake --build . -- -j2
- SIMDJSON_FORCE_IMPLEMENTATION=ppc64 ctest $CTEST_FLAGS -L per_implementation
- SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation
+1 -12
View File
@@ -85,17 +85,6 @@
"valarray": "cpp",
"vector": "cpp",
"*.ipp": "cpp",
"__functional_base_03": "cpp",
"filesystem": "cpp",
"*.inc": "cpp",
"compare": "cpp",
"concepts": "cpp",
"variant": "cpp",
"__bits": "cpp",
"csignal": "cpp",
"future": "cpp",
"queue": "cpp",
"shared_mutex": "cpp",
"ranges": "cpp"
"filesystem": "cpp"
}
}
+13 -23
View File
@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
project(
simdjson
# The version number is modified by tools/release.py
VERSION 3.0.1
VERSION 0.9.1
DESCRIPTION "Parsing gigabytes of JSON per second"
HOMEPAGE_URL "https://simdjson.org/"
LANGUAGES CXX C
@@ -20,8 +20,8 @@ string(
# ---- Options, variables ----
# These version numbers are modified by tools/release.py
set(SIMDJSON_LIB_VERSION "14.0.0" CACHE STRING "simdjson library version")
set(SIMDJSON_LIB_SOVERSION "14" CACHE STRING "simdjson library soversion")
set(SIMDJSON_LIB_VERSION "8.0.0" CACHE STRING "simdjson library version")
set(SIMDJSON_LIB_SOVERSION "8" CACHE STRING "simdjson library soversion")
option(SIMDJSON_ENABLE_THREADS "Link with thread support" ON)
@@ -49,10 +49,19 @@ endif()
if(is_top_project)
option(SIMDJSON_DEVELOPER_MODE "Enable targets for developing simdjson" OFF)
option(BUILD_SHARED_LIBS "Build simdjson as a shared library" OFF)
if("$ENV{CI}")
set(SIMDJSON_DEVELOPER_MODE ON CACHE INTERNAL "")
endif()
endif()
include(cmake/handle-deprecations.cmake)
include(cmake/developer-options.cmake)
if(SIMDJSON_DEVELOPER_MODE)
include(cmake/developer-options.cmake)
else()
message(STATUS "Building only the library. Advanced users may want to turn SIMDJSON_DEVELOPER_MODE to ON, e.g., via -D SIMDJSON_DEVELOPER_MODE=ON.")
endif()
# ---- simdjson library ----
@@ -152,24 +161,6 @@ install(
COMPONENT example_Development
)
# pkg-config
include(cmake/JoinPaths.cmake)
join_paths(PKGCONFIG_INCLUDEDIR "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}")
join_paths(PKGCONFIG_LIBDIR "\${prefix}" "${CMAKE_INSTALL_LIBDIR}")
if(SIMDJSON_ENABLE_THREADS)
set(PKGCONFIG_CFLAGS "-DSIMDJSON_THREADS_ENABLED=1")
if(CMAKE_THREAD_LIBS_INIT)
set(PKGCONFIG_LIBS_PRIVATE "Libs.private: ${CMAKE_THREAD_LIBS_INIT}")
endif()
endif()
configure_file("simdjson.pc.in" "simdjson.pc" @ONLY)
install(
FILES "${CMAKE_CURRENT_BINARY_DIR}/simdjson.pc"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig"
)
#
# CPack
#
@@ -189,7 +180,6 @@ endif()
# ---- Developer mode extras ----
if(NOT SIMDJSON_DEVELOPER_MODE)
message(STATUS "Building only the library. Advanced users may want to turn SIMDJSON_DEVELOPER_MODE to ON, e.g., via -D SIMDJSON_DEVELOPER_MODE=ON.")
return()
elseif(NOT is_top_project)
message(AUTHOR_WARNING "Developer mode is intended for developers of simdjson")
+5 -5
View File
@@ -54,7 +54,7 @@ Contributors are encouraged to :
- Document their changes. Though we do not enforce a rule regarding code comments, we prefer that non-trivial algorithms and techniques be somewhat documented in the code.
- Follow as much as possible the existing code style. We do not enforce a specific code style, but we prefer consistency.
- Modify as few lines of code as possible when working on an issue. The more lines you modify, the harder it is for your fellow human beings to understand what is going on.
- Tools may report "problems" with the code, but we never delegate programming to tools: if there is a problem with the code, we need to understand it. Thus we will not "fix" code merely to please a static analyzer.
- Tools may report "problems" with the code, but we never delegate programming to tools: if there is a problem with the code, we need to understand it. Thus we will not "fix" code merely to please a static analyzer if we do not understand.
- Provide tests for any new feature. We will not merge a new feature without tests.
Pull Requests
@@ -62,13 +62,13 @@ Pull Requests
Pull requests are always invited. However, we ask that you follow these guidelines:
- It is wise to discuss your ideas first as part of an issue before you start coding. If you omit this step and code first, be prepared to have your code receive scrutiny and be dropped.
- Users should provide a rationale for their changes. Does it improve performance? Does it add a feature? Does it improve maintainability? Does it fix a bug? This must be explicitly stated as part of the pull request. Do not propose changes based on taste or intuition. We do not delegate programming to tools: that some tool suggested a code change is not reason enough to change the code.
- It is wiser to discuss your ideas first as part of an issue before you start coding. If you omit this step and code first, be prepare to have your code receive scrutiny and be dropped.
- Users should provide a rationale for their changes. Does it improve performance? Does it add a feature? Does it improve maintainability? Does fix a bug? This must be explicitly stated as part of the pull request. Do not propose changes based on taste or intuition. We do not delegate programming to tools: that some tool suggested a code change is not reason enough to change the code.
1. When your code improves performance, please document the gains with a benchmark using hard numbers.
2. If your code fixes a bug, please either fix a failing test, or propose a new test.
2. If your code fixes a bug, please be either fix a failing test, or propose a new test.
3. Other types of changes must be clearly motivated. We openly discourage changes with no identifiable benefits.
- Changes should be focused and minimal. You should change as few lines of code as possible. Please do not reformat or touch files needlessly.
- New features must be accompanied by new tests, in general.
- New features must be accompanied of new tests, in general.
- Your code should pass our continuous-integration tests. It is your responsibility to ensure that your proposal pass the tests. We do not merge pull requests that would break our build.
- An exception to this would be changes to non-code files, such as documentation and assets, or trivial changes to code, such as comments, where it is encouraged to explicitly ask for skipping a CI run using the `[skip ci]` prefix in your Pull Request title **and** in the first line of the most recent commit in a push. Example for such a commit: `[skip ci] Fixed typo in power_of_ten's docs`
This benefits the project in such a way that the CI pipeline is not burdened by running jobs on changes that don't change any behavior in the code, which reduces wait times for other Pull Requests that do change behavior and require testing.
+1 -5
View File
@@ -37,9 +37,5 @@ Furkan Taşkale
Brendan Knapp
Danila Kutenin
Pavel Pavlov
Hao Chen
Nicolas Boyer
Kim Walisch and Jatin Bhateja (AVX-512 bitset decoder)
Fangzheng Zhang and Weiqiang Wan (AVX-512 kernel)
# if you have contributed to the project and your name does not
# if you have contributed to the project and your name does not
# appear in this list, please let us know!
+1 -1
View File
@@ -38,7 +38,7 @@ PROJECT_NAME = simdjson
# could be handy for archiving the generated documentation or if some version
# control system is used.
PROJECT_NUMBER = "3.0.1"
PROJECT_NUMBER = "0.9.1"
# Using the PROJECT_BRIEF tag one can provide an optional one line description
# for a project that appears at the top of each page and should give viewer a
+2 -2
View File
@@ -82,7 +82,7 @@ Other important files and directories:
* **singleheader/amalgamate.py:** Generates `singleheader/simdjson.h` and `singleheader/simdjson.cpp` for release (python script).
* **benchmark:** This is where we do benchmarking. Benchmarking is core to every change we make; the
cardinal rule is don't regress performance without knowing exactly why, and what you're trading
for it. Many of our benchmarks are microbenchmarks. We are effectively doing controlled scientific experiments for the purpose of understanding what affects our performance. So we simplify as much as possible. We try to avoid irrelevant factors such as page faults, interrupts, unnecessary system calls. We recommend checking the performance as follows:
for it. Many of our benchmarks are microbenchmarks. We are effectively doing controlled scientific experiments for the purpose of understanding what affects our performance. So we simplify as much as possible. We try to avoid irrelevant factors such as page faults, interrupts, unnnecessary system calls. We recommend checking the performance as follows:
```bash
mkdir build
cd build
@@ -114,7 +114,7 @@ Other important files and directories:
* `json2json mydoc.json` parses the document, constructs a model and then dumps back the result to standard output.
* `json2json -d mydoc.json` parses the document, constructs a model and then dumps model (as a tape) to standard output. The tape format is described in the accompanying file `tape.md`.
* `minify mydoc.json` minifies the JSON document, outputting the result to standard output. Minifying means to remove the unneeded white space characters.
* `jsonpointer mydoc.json <jsonpath> <jsonpath> ... <jsonpath>` parses the document, constructs a model and then processes a series of [JSON Pointer paths](https://tools.ietf.org/html/rfc6901). The result is itself a JSON document.
*`jsonpointer mydoc.json <jsonpath> <jsonpath> ... <jsonpath>` parses the document, constructs a model and then processes a series of [JSON Pointer paths](https://tools.ietf.org/html/rfc6901). The result is itself a JSON document.
> **Don't modify the files in singleheader/ directly; these are automatically generated.**
+6 -10
View File
@@ -3,7 +3,7 @@
[![Ubuntu 20.04 CI](https://github.com/simdjson/simdjson/workflows/Ubuntu%2020.04%20CI%20(GCC%209)/badge.svg)](https://simdjson.org/plots.html)
![VS16-CI](https://github.com/simdjson/simdjson/workflows/VS16-CI/badge.svg)
![MinGW64-CI](https://github.com/simdjson/simdjson/workflows/MinGW64-CI/badge.svg)
[![][license img]][license] [![Doxygen Documentation](https://img.shields.io/badge/docs-doxygen-green.svg)](https://simdjson.org/api/3.0.0/index.html)
[![][license img]][license] [![Doxygen Documentation](https://img.shields.io/badge/docs-doxygen-green.svg)](https://simdjson.org/api/0.9.0/index.html)
simdjson : Parsing gigabytes of JSON per second
===============================================
@@ -53,7 +53,6 @@ The simdjson library is easily consumable with a single .h and .cpp file.
2. Create `quickstart.cpp`:
```c++
#include <iostream>
#include "simdjson.h"
using namespace simdjson;
int main(void) {
@@ -79,7 +78,7 @@ Usage documentation is available:
* [Performance](doc/performance.md) shows some more advanced scenarios and how to tune for them.
* [Implementation Selection](doc/implementation-selection.md) describes runtime CPU detection and
how you can work with it.
* [API](https://simdjson.org/api/1.0.0/annotated.html) contains the automatically generated API documentation.
* [API](https://simdjson.org/api/0.9.0/annotated.html) contains the automatically generated API documentation.
Performance results
-------------------
@@ -111,12 +110,10 @@ For NDJSON files, we can exceed 3 GB/s with [our multithreaded parsing function
Real-world usage
----------------
- [ClickHouse](https://github.com/ClickHouse/ClickHouse)
- [Microsoft FishStore](https://github.com/microsoft/FishStore)
- [Yandex ClickHouse](https://github.com/yandex/ClickHouse)
- [Clang Build Analyzer](https://github.com/aras-p/ClangBuildAnalyzer)
- [Shopify HeapProfiler](https://github.com/Shopify/heap-profiler)
- [StarRocks](https://github.com/StarRocks/starrocks)
- [Microsoft FishStore](https://github.com/microsoft/FishStore)
If you are planning to use simdjson in a product, please work from one of our releases.
@@ -140,8 +137,7 @@ We distinguish between "bindings" (which just wrap the C++ code) and a port to a
- [rcppsimdjson](https://github.com/eddelbuettel/rcppsimdjson): R bindings.
- [simdjson_erlang](https://github.com/ChomperT/simdjson_erlang): erlang bindings.
- [lua-simdjson](https://github.com/FourierTransformer/lua-simdjson): lua bindings.
- [hermes-json](https://hackage.haskell.org/package/hermes-json): haskell bindings.
- [simdjzon](https://github.com/travisstaloch/simdjzon): zig port.
About simdjson
--------------
@@ -162,7 +158,7 @@ We also have an informal [blog post providing some background and context](https
For the video inclined, <br />
[![simdjson at QCon San Francisco 2019](http://img.youtube.com/vi/wlvKAT7SZIQ/0.jpg)](http://www.youtube.com/watch?v=wlvKAT7SZIQ)<br />
(It was the best voted talk, we're kinda proud of it.)
(it was the best voted talk, we're kinda proud of it).
Funding
-------
@@ -4,22 +4,18 @@
#include <map>
#include <string>
namespace amazon_cellphones {
const bool UNTHREADED = false;
const bool THREADED = true;
using namespace json_benchmark;
struct brand {
double cumulative_rating;
uint64_t reviews_count;
simdjson_inline bool operator==(const brand &other) const {
simdjson_really_inline bool operator==(const brand &other) const {
return cumulative_rating == other.cumulative_rating &&
reviews_count == other.reviews_count;
}
simdjson_inline bool operator!=(const brand &other) const { return !(*this == other); }
simdjson_really_inline bool operator!=(const brand &other) const { return !(*this == other); }
};
simdjson_unused static std::ostream &operator<<(std::ostream &o, const brand &b) {
@@ -63,11 +59,10 @@ struct runner : public file_runner<I> {
}
};
template<bool threaded>
struct simdjson_dom;
template<typename I> simdjson_inline static void amazon_cellphones(benchmark::State &state) {
run_json_benchmark<runner<I>, runner<simdjson_dom<UNTHREADED>>>(state);
template<typename I> simdjson_really_inline static void amazon_cellphones(benchmark::State &state) {
run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
}
} // namespace amazon_cellphones
} // namespace amazon_cellphones
+1 -8
View File
@@ -8,16 +8,12 @@ namespace amazon_cellphones {
using namespace simdjson;
template<bool threaded>
struct simdjson_dom {
using StringType = std::string;
dom::parser parser{};
bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
#ifdef SIMDJSON_THREADS_ENABLED
parser.threaded = threaded;
#endif
auto stream = parser.parse_many(json);
auto i = stream.begin();
++i; // Skip first line
@@ -41,10 +37,7 @@ struct simdjson_dom {
};
BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_dom<UNTHREADED>)->UseManualTime();
#ifdef SIMDJSON_THREADS_ENABLED
BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_dom<THREADED>)->UseManualTime();
#endif
BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_dom)->UseManualTime();
} // namespace amazon_cellphones
@@ -8,21 +8,17 @@ namespace amazon_cellphones {
using namespace simdjson;
template<bool threaded>
struct simdjson_ondemand {
using StringType = std::string;
ondemand::parser parser{};
bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
#ifdef SIMDJSON_THREADS_ENABLED
parser.threaded = threaded;
#endif
ondemand::document_stream stream = parser.iterate_many(json);
ondemand::document_stream::iterator i = stream.begin();
++i; // Skip first line
for (;i != stream.end(); ++i) {
auto doc = *i;
auto & doc = *i;
size_t index{0};
StringType copy;
double rating;
@@ -62,10 +58,7 @@ struct simdjson_ondemand {
};
BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_ondemand<UNTHREADED>)->UseManualTime();
#ifdef SIMDJSON_THREADS_ENABLED
BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_ondemand<THREADED>)->UseManualTime();
#endif
BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_ondemand)->UseManualTime();
} // namespace amazon_cellphones
-26
View File
@@ -1,5 +1,4 @@
#include <benchmark/benchmark.h>
#include <iostream>
#include "simdjson.h"
#include <sstream>
@@ -656,31 +655,6 @@ static void error_code_twitter_image_sizes(State& state) noexcept {
}
BENCHMARK(error_code_twitter_image_sizes);
static void parse_surrogate_pairs(State& state) {
// NOTE: This mostly exists to show there's a tiny benefit to
// loading and comparing both bytes of "\\u" simultaneously.
// (which should also reduce the compiled code size).
// The repeated surrogate pairs make this easier to measure.
dom::parser parser;
const std::string_view data = "\"\\uD834\\uDD1E\\uD834\\uDD1E\\uD834\\uDD1E\\uD834\\uDD1E\\uD834\\uDD1E\\uD834\\uDD1E\\uD834\\uDD1E\\uD834\\uDD1E\\uD834\\uDD1E\\uD834\\uDD1E\"";
padded_string docdata{data};
// we do not want mem. alloc. in the loop.
auto error = parser.allocate(docdata.size());
if (error) {
cout << error << endl;
return;
}
for (simdjson_unused auto _ : state) {
dom::element doc;
if ((error = parser.parse(docdata).get(doc))) {
cerr << "could not parse string" << error << endl;
return;
}
}
}
BENCHMARK(parse_surrogate_pairs);
#ifndef SIMDJSON_DISABLE_DEPRECATED_API
SIMDJSON_PUSH_DISABLE_WARNINGS
+57 -105
View File
@@ -25,111 +25,6 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
#include <benchmark/benchmark.h>
SIMDJSON_POP_DISABLE_WARNINGS
#include "json2msgpack/simdjson_ondemand.h"
#include "json2msgpack/simdjson_dom.h"
#include "json2msgpack/yyjson.h"
#include "json2msgpack/rapidjson.h"
#if SIMDJSON_COMPETITION_ONDEMAND_SAJSON
#include "json2msgpack/sajson.h"
#endif // SIMDJSON_COMPETITION_ONDEMAND_SAJSON
#include "json2msgpack/nlohmann_json.h"
#include "partial_tweets/simdjson_ondemand.h"
#include "partial_tweets/simdjson_dom.h"
#include "partial_tweets/yyjson.h"
#if SIMDJSON_COMPETITION_ONDEMAND_SAJSON
#include "partial_tweets/sajson.h"
#endif // SIMDJSON_COMPETITION_ONDEMAND_SAJSON
#include "partial_tweets/rapidjson.h"
#if SIMDJSON_COMPETITION_SAX
#include "partial_tweets/rapidjson_sax.h"
#endif // SIMDJSON_COMPETITION_SAX
#include "partial_tweets/nlohmann_json.h"
#if SIMDJSON_COMPETITION_SAX
#include "partial_tweets/nlohmann_json_sax.h"
#endif // SIMDJSON_COMPETITION_SAX
#include "distinct_user_id/simdjson_ondemand.h"
#include "distinct_user_id/simdjson_ondemand_json_pointer.h"
#include "distinct_user_id/simdjson_dom.h"
#include "distinct_user_id/simdjson_dom_json_pointer.h"
#include "distinct_user_id/yyjson.h"
#if SIMDJSON_COMPETITION_ONDEMAND_SAJSON
#include "distinct_user_id/sajson.h"
#endif // SIMDJSON_COMPETITION_ONDEMAND_SAJSON
#include "distinct_user_id/rapidjson.h"
#if SIMDJSON_COMPETITION_SAX
#include "distinct_user_id/rapidjson_sax.h"
#endif // SIMDJSON_COMPETITION_SAX
#include "distinct_user_id/nlohmann_json.h"
#if SIMDJSON_COMPETITION_SAX
#include "distinct_user_id/nlohmann_json_sax.h"
#endif // SIMDJSON_COMPETITION_SAX
#include "find_tweet/simdjson_ondemand.h"
#include "find_tweet/simdjson_dom.h"
#include "find_tweet/yyjson.h"
#if SIMDJSON_COMPETITION_ONDEMAND_SAJSON
#include "find_tweet/sajson.h"
#endif // SIMDJSON_COMPETITION_ONDEMAND_SAJSON
#include "find_tweet/rapidjson.h"
#if SIMDJSON_COMPETITION_SAX
#include "find_tweet/rapidjson_sax.h"
#endif // SIMDJSON_COMPETITION_SAX
#include "find_tweet/nlohmann_json.h"
#if SIMDJSON_COMPETITION_SAX
#include "find_tweet/nlohmann_json_sax.h"
#endif // SIMDJSON_COMPETITION_SAX
#include "top_tweet/simdjson_ondemand.h"
#include "top_tweet/simdjson_dom.h"
#include "top_tweet/yyjson.h"
#if SIMDJSON_COMPETITION_ONDEMAND_SAJSON
#include "top_tweet/sajson.h"
#endif // SIMDJSON_COMPETITION_ONDEMAND_SAJSON
#include "top_tweet/rapidjson.h"
#if SIMDJSON_COMPETITION_SAX
#include "top_tweet/rapidjson_sax.h"
#endif // SIMDJSON_COMPETITION_SAX
#include "top_tweet/nlohmann_json.h"
#if SIMDJSON_COMPETITION_SAX
#include "top_tweet/nlohmann_json_sax.h"
#endif // SIMDJSON_COMPETITION_SAX
#include "kostya/simdjson_ondemand.h"
#include "kostya/simdjson_dom.h"
#include "kostya/yyjson.h"
#if SIMDJSON_COMPETITION_ONDEMAND_SAJSON
#include "kostya/sajson.h"
#endif // SIMDJSON_COMPETITION_ONDEMAND_SAJSON
#include "kostya/rapidjson.h"
#if SIMDJSON_COMPETITION_SAX
#include "kostya/rapidjson_sax.h"
#endif // SIMDJSON_COMPETITION_SAX
#include "kostya/nlohmann_json.h"
#if SIMDJSON_COMPETITION_SAX
#include "kostya/nlohmann_json_sax.h"
#endif // SIMDJSON_COMPETITION_SAX
#include "large_random/simdjson_ondemand.h"
#if SIMDJSON_COMPETITION_ONDEMAND_UNORDERED
#include "large_random/simdjson_ondemand_unordered.h"
#endif // SIMDJSON_COMPETITION_ONDEMAND_UNORDERED
#include "large_random/simdjson_dom.h"
#include "large_random/yyjson.h"
#if SIMDJSON_COMPETITION_ONDEMAND_SAJSON
#include "large_random/sajson.h"
#endif // SIMDJSON_COMPETITION_ONDEMAND_SAJSON
#include "large_random/rapidjson.h"
#if SIMDJSON_COMPETITION_SAX
#include "large_random/rapidjson_sax.h"
#endif // SIMDJSON_COMPETITION_SAX
#include "large_random/nlohmann_json.h"
#if SIMDJSON_COMPETITION_SAX
#include "large_random/nlohmann_json_sax.h"
#endif // SIMDJSON_COMPETITION_SAX
#include "amazon_cellphones/simdjson_dom.h"
#include "amazon_cellphones/simdjson_ondemand.h"
@@ -137,4 +32,61 @@ SIMDJSON_POP_DISABLE_WARNINGS
#include "large_amazon_cellphones/simdjson_dom.h"
#include "large_amazon_cellphones/simdjson_ondemand.h"
#include "partial_tweets/simdjson_dom.h"
#include "partial_tweets/simdjson_ondemand.h"
#include "partial_tweets/yyjson.h"
#include "partial_tweets/sajson.h"
#include "partial_tweets/rapidjson.h"
#include "partial_tweets/rapidjson_sax.h"
#include "partial_tweets/nlohmann_json.h"
#include "partial_tweets/nlohmann_json_sax.h"
#include "large_random/simdjson_dom.h"
#include "large_random/simdjson_ondemand.h"
#include "large_random/simdjson_ondemand_unordered.h"
#include "large_random/yyjson.h"
#include "large_random/sajson.h"
#include "large_random/rapidjson.h"
#include "large_random/rapidjson_sax.h"
#include "large_random/nlohmann_json.h"
#include "large_random/nlohmann_json_sax.h"
#include "kostya/simdjson_dom.h"
#include "kostya/simdjson_ondemand.h"
#include "kostya/yyjson.h"
#include "kostya/sajson.h"
#include "kostya/rapidjson.h"
#include "kostya/rapidjson_sax.h"
#include "kostya/nlohmann_json.h"
#include "kostya/nlohmann_json_sax.h"
#include "distinct_user_id/simdjson_dom.h"
#include "distinct_user_id/simdjson_dom_json_pointer.h"
#include "distinct_user_id/simdjson_ondemand.h"
#include "distinct_user_id/simdjson_ondemand_json_pointer.h"
#include "distinct_user_id/yyjson.h"
#include "distinct_user_id/sajson.h"
#include "distinct_user_id/rapidjson.h"
#include "distinct_user_id/rapidjson_sax.h"
#include "distinct_user_id/nlohmann_json.h"
#include "distinct_user_id/nlohmann_json_sax.h"
#include "find_tweet/simdjson_dom.h"
#include "find_tweet/simdjson_ondemand.h"
#include "find_tweet/yyjson.h"
#include "find_tweet/sajson.h"
#include "find_tweet/rapidjson.h"
#include "find_tweet/rapidjson_sax.h"
#include "find_tweet/nlohmann_json.h"
#include "find_tweet/nlohmann_json_sax.h"
#include "top_tweet/simdjson_dom.h"
#include "top_tweet/simdjson_ondemand.h"
#include "top_tweet/yyjson.h"
#include "top_tweet/sajson.h"
#include "top_tweet/rapidjson.h"
#include "top_tweet/rapidjson_sax.h"
#include "top_tweet/nlohmann_json.h"
#include "top_tweet/nlohmann_json_sax.h"
BENCHMARK_MAIN();
-1
View File
@@ -1,5 +1,4 @@
#include <benchmark/benchmark.h>
#include <iostream>
#include "simdjson.h"
using namespace simdjson;
using namespace benchmark;
+6 -42
View File
@@ -97,9 +97,9 @@ struct option_struct {
verbose = true;
break;
case 'a': {
auto impl = simdjson::get_available_implementations()[optarg];
auto impl = simdjson::available_implementations[optarg];
if(impl && impl->supported_by_runtime_system()) {
simdjson::get_active_implementation() = impl;
simdjson::active_implementation = impl;
} else {
std::cerr << "implementation " << optarg << " not found or not supported " << std::endl;
}
@@ -164,7 +164,7 @@ struct feature_benchmarker {
}
simdjson_inline void run_iterations(size_t iterations, bool stage1_only=false) {
simdjson_really_inline void run_iterations(size_t iterations, bool stage1_only=false) {
struct7.run_iterations(iterations, stage1_only);
struct7_miss.run_iterations(iterations, stage1_only);
struct7_full.run_iterations(iterations, stage1_only);
@@ -204,13 +204,10 @@ struct feature_benchmarker {
}
// Rate of 1-7-structural misses per 8-structural flip
double struct1_7_miss_rate(BenchmarkStage stage) const {
#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
return 1;
#else
if (!has_events()) { return 1; }
return struct7_miss[stage].best.branch_misses() - struct7[stage].best.branch_misses() / double(struct7_miss.stats->blocks_with_1_structural_flipped);
#endif
}
// Extra cost of an 8-15 structural block over a 1-7 structural block
double struct8_15_cost(BenchmarkStage stage) const {
return cost_per_block(stage, struct15, struct15.stats->blocks_with_8_structurals, struct7);
@@ -221,12 +218,8 @@ struct feature_benchmarker {
}
// Rate of 8-15-structural misses per 8-structural flip
double struct8_15_miss_rate(BenchmarkStage stage) const {
#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
return 1;
#else
if (!has_events()) { return 1; }
return double(struct15_miss[stage].best.branch_misses() - struct15[stage].best.branch_misses()) / double(struct15_miss.stats->blocks_with_8_structurals_flipped);
#endif
}
// Extra cost of a 16+-structural block over an 8-15 structural block (actual varies based on # of structurals!)
@@ -239,15 +232,10 @@ struct feature_benchmarker {
}
// Rate of 16-structural misses per 16-structural flip
double struct16_miss_rate(BenchmarkStage stage) const {
#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
return 1;
#else
if (!has_events()) { return 1; }
return double(struct23_miss[stage].best.branch_misses() - struct23[stage].best.branch_misses()) / double(struct23_miss.stats->blocks_with_16_structurals_flipped);
#endif
}
// Extra cost of having UTF-8 in a block
double utf8_cost(BenchmarkStage stage) const {
return cost_per_block(stage, utf8, utf8.stats->blocks_with_utf8, struct7_full);
@@ -258,13 +246,10 @@ struct feature_benchmarker {
}
// Rate of UTF-8 misses per UTF-8 flip
double utf8_miss_rate(BenchmarkStage stage) const {
#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
return 1;
#else
if (!has_events()) { return 1; }
return double(utf8_miss[stage].best.branch_misses() - utf8[stage].best.branch_misses()) / double(utf8_miss.stats->blocks_with_utf8_flipped);
#endif
}
// Extra cost of having escapes in a block
double escape_cost(BenchmarkStage stage) const {
return cost_per_block(stage, escape, escape.stats->blocks_with_escapes, struct7_full);
@@ -275,15 +260,10 @@ struct feature_benchmarker {
}
// Rate of escape misses per escape flip
double escape_miss_rate(BenchmarkStage stage) const {
#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
return 1;
#else
if (!has_events()) { return 1; }
return double(escape_miss[stage].best.branch_misses() - escape[stage].best.branch_misses()) / double(escape_miss.stats->blocks_with_escapes_flipped);
#endif
}
double calc_expected_feature_cost(BenchmarkStage stage, const benchmarker& file) const {
// Expected base ns/block (empty)
json_stats& stats = *file.stats;
@@ -320,6 +300,7 @@ struct feature_benchmarker {
double calc_expected(BenchmarkStage stage, const benchmarker& file) const {
return calc_expected_feature_cost(stage, file) + calc_expected_miss_cost(stage, file);
}
void print(const option_struct& options) const {
printf("\n");
printf("Features in ns/block (64 bytes):\n");
@@ -378,22 +359,6 @@ struct feature_benchmarker {
}
};
#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
void print_file_effectiveness(BenchmarkStage stage, const char* filename, const benchmarker& results, const feature_benchmarker& features) {
double actual = results[stage].best.elapsed_ns() / double(results.stats->blocks);
double calc = features.calc_expected(stage, results);
double calc_misses = features.calc_expected_misses(stage, results);
double calc_miss_cost = features.calc_expected_miss_cost(stage, results);
printf(" | %-8s ", benchmark_stage_name(stage));
printf("| %-15s ", filename);
printf("| %8.3g ", features.calc_expected_feature_cost(stage, results));
printf("| %8.3g ", calc_miss_cost);
printf("| %8.3g ", calc);
printf("| %8.3g ", actual);
printf("| %+8.3g ", actual - calc);
printf("| %13llu ", (long long unsigned)(calc_misses));
}
#else
void print_file_effectiveness(BenchmarkStage stage, const char* filename, const benchmarker& results, const feature_benchmarker& features) {
double actual = results[stage].best.elapsed_ns() / double(results.stats->blocks);
double calc = features.calc_expected(stage, results);
@@ -417,7 +382,6 @@ void print_file_effectiveness(BenchmarkStage stage, const char* filename, const
}
printf("|\n");
}
#endif
int main(int argc, char *argv[]) {
// Read options
+4 -5
View File
@@ -2,7 +2,7 @@
#define __BENCHMARKER_H
#include "event_counter.h"
#include "simdjson.h"
#include "simdjson.h" // For SIMDJSON_DISABLE_DEPRECATED_WARNINGS
#include <cassert>
#include <cctype>
@@ -308,7 +308,7 @@ struct benchmarker {
return all_stages_without_allocation.iterations;
}
simdjson_inline void run_iteration(bool stage1_only, bool hotbuffers=false) {
simdjson_really_inline void run_iteration(bool stage1_only, bool hotbuffers=false) {
// Allocate dom::parser
collector.start();
dom::parser parser;
@@ -384,7 +384,7 @@ struct benchmarker {
loop << all_loop_count;
}
simdjson_inline void run_iterations(size_t iterations, bool stage1_only, bool hotbuffers=false) {
simdjson_really_inline void run_iterations(size_t iterations, bool stage1_only, bool hotbuffers=false) {
for (size_t i = 0; i<iterations; i++) {
run_iteration(stage1_only, hotbuffers);
}
@@ -423,7 +423,7 @@ struct benchmarker {
stage.instructions() / static_cast<double>(stats->structurals),
stage.instructions() / static_cast<double>(stage.cycles())
);
#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
// NOTE: removed cycles/miss because it is a somewhat misleading stat
printf("%s%-13s: %7.0f branch misses (%6.2f%%) - %.0f cache misses (%6.2f%%) - %.2f cache references\n",
prefix,
@@ -434,7 +434,6 @@ struct benchmarker {
percent(stage.cache_misses(), all_stages_without_allocation.cache_misses()),
stage.cache_references()
);
#endif
}
}
@@ -46,7 +46,7 @@ struct runner : public file_runner<I> {
struct simdjson_dom;
template<typename I> simdjson_inline static void distinct_user_id(benchmark::State &state) {
template<typename I> simdjson_really_inline static void distinct_user_id(benchmark::State &state) {
run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
}
+2 -3
View File
@@ -46,14 +46,13 @@ struct rapidjson : rapidjson_base {
};
BENCHMARK_TEMPLATE(distinct_user_id, rapidjson)->UseManualTime();
#if SIMDJSON_COMPETITION_ONDEMAND_INSITU
struct rapidjson_insitu : rapidjson_base {
bool run(simdjson::padded_string &json, std::vector<uint64_t> &result) {
return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), result);
return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), result);
}
};
BENCHMARK_TEMPLATE(distinct_user_id, rapidjson_insitu)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_INSITU
} // namespace partial_tweets
#endif // SIMDJSON_COMPETITION_RAPIDJSON
+2 -4
View File
@@ -9,14 +9,12 @@ namespace distinct_user_id {
struct sajson {
size_t ast_buffer_size{0};
size_t *ast_buffer{nullptr};
~sajson() { free(ast_buffer); }
simdjson_inline std::string_view get_string_view(const ::sajson::value &obj, std::string_view key) {
simdjson_really_inline std::string_view get_string_view(const ::sajson::value &obj, std::string_view key) {
auto val = obj.get_value_of_key({key.data(), key.length()});
if (val.get_type() != ::sajson::TYPE_STRING) { throw "field is not a string"; }
return { val.as_cstring(), val.get_string_length() };
}
simdjson_inline uint64_t get_str_uint64(const ::sajson::value &obj, std::string_view key) {
simdjson_really_inline uint64_t get_str_uint64(const ::sajson::value &obj, std::string_view key) {
// Since sajson only supports 53-bit numbers, and IDs in twitter.json can be > 53 bits, we read the corresponding id_str and parse that.
auto val = obj.get_value_of_key({key.data(), key.length()});
if (val.get_type() != ::sajson::TYPE_STRING) { throw "field not a string"; }
+1 -2
View File
@@ -49,14 +49,13 @@ struct yyjson : yyjson_base {
};
BENCHMARK_TEMPLATE(distinct_user_id, yyjson)->UseManualTime();
#if SIMDJSON_COMPETITION_ONDEMAND_INSITU
struct yyjson_insitu : yyjson_base {
bool run(simdjson::padded_string &json, std::vector<uint64_t> &result) {
return yyjson_base::run(yyjson_read_opts(json.data(), json.size(), YYJSON_READ_INSITU, 0, 0), result);
}
};
BENCHMARK_TEMPLATE(distinct_user_id, yyjson_insitu)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_INSITU
} // namespace distinct_user_id
#endif // SIMDJSON_COMPETITION_YYJSON
+5 -5
View File
@@ -66,7 +66,7 @@ void print_usage(ostream& out) {
out << "-H - Make the buffers hot (reduce page allocation and related OS tasks during parsing) [default]" << endl;
out << "-a IMPL - Use the given parser implementation. By default, detects the most advanced" << endl;
out << " implementation supported on the host machine." << endl;
for (auto impl : simdjson::get_available_implementations()) {
for (auto impl : simdjson::available_implementations) {
if(impl->supported_by_runtime_system()) {
out << "-a " << std::left << std::setw(9) << impl->name() << " - Use the " << impl->description() << " parser implementation." << endl;
}
@@ -116,10 +116,10 @@ struct option_struct {
verbose = true;
break;
case 'a': {
const implementation *impl = simdjson::get_available_implementations()[optarg];
const implementation *impl = simdjson::available_implementations[optarg];
if ((!impl) || (!impl->supported_by_runtime_system())) {
std::string exit_message = string("Unsupported option value -a ") + optarg + ": expected -a with one of ";
for (auto imple : simdjson::get_available_implementations()) {
for (auto imple : simdjson::available_implementations) {
if(imple->supported_by_runtime_system()) {
exit_message += imple->name();
exit_message += " ";
@@ -127,7 +127,7 @@ struct option_struct {
}
exit_usage(exit_message);
}
simdjson::get_active_implementation() = impl;
simdjson::active_implementation = impl;
break;
}
case 'C':
@@ -175,7 +175,7 @@ int main(int argc, char *argv[]) {
option_struct options(argc, argv);
if (options.verbose) {
verbose_stream = &cout;
verbose() << "Implementation: " << simdjson::get_active_implementation()->name() << endl;
verbose() << "Implementation: " << simdjson::active_implementation->name() << endl;
}
// Start collecting events. We put this early so if it prints an error message, it's the
+3 -3
View File
@@ -41,7 +41,7 @@ using stat_t = struct stat_s;
simdjson_inline void simdjson_process_atom(stat_t &s,
simdjson_really_inline void simdjson_process_atom(stat_t &s,
simdjson::dom::element element) {
if (element.is<int64_t>()) {
s.integer_count++;
@@ -50,9 +50,9 @@ simdjson_inline void simdjson_process_atom(stat_t &s,
} else if(element.is<double>()) {
s.float_count++;
} else if (element.is<bool>()) {
simdjson::error_code err;
bool v;
simdjson::error_code error;
if ((error = element.get(v))) { std::cerr << error << std::endl; abort(); }
err = element.get(v);
if (v) {
s.true_count++;
} else {
+4 -27
View File
@@ -1,15 +1,6 @@
#ifndef __EVENT_COUNTER_H
#define __EVENT_COUNTER_H
#ifndef SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
#ifdef __aarch64__
// on ARM, we use just cycles and instructions
#define SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS 1
#else
// elsewhere, we try to use four counters.
#define SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS 0
#endif
#endif
#include <cassert>
#include <cctype>
#ifndef _MSC_VER
@@ -55,12 +46,6 @@ struct event_count {
event_count(const event_count& other): elapsed(other.elapsed), event_counts(other.event_counts) { }
// The types of counters (so we can read the getter more easily)
#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
enum event_counter_types {
CPU_CYCLES,
INSTRUCTIONS
};
#else
enum event_counter_types {
CPU_CYCLES,
INSTRUCTIONS,
@@ -68,16 +53,15 @@ struct event_count {
CACHE_REFERENCES,
CACHE_MISSES
};
#endif
double elapsed_sec() const { return duration<double>(elapsed).count(); }
double elapsed_ns() const { return duration<double, std::nano>(elapsed).count(); }
double cycles() const { return static_cast<double>(event_counts[CPU_CYCLES]); }
double instructions() const { return static_cast<double>(event_counts[INSTRUCTIONS]); }
#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
double branch_misses() const { return static_cast<double>(event_counts[BRANCH_MISSES]); }
double cache_references() const { return static_cast<double>(event_counts[CACHE_REFERENCES]); }
double cache_misses() const { return static_cast<double>(event_counts[CACHE_MISSES]); }
#endif
event_count& operator=(const event_count& other) {
this->elapsed = other.elapsed;
this->event_counts = other.event_counts;
@@ -121,11 +105,9 @@ struct event_aggregate {
double elapsed_ns() const { return total.elapsed_ns() / iterations; }
double cycles() const { return total.cycles() / iterations; }
double instructions() const { return total.instructions() / iterations; }
#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
double branch_misses() const { return total.branch_misses() / iterations; }
double cache_references() const { return total.cache_references() / iterations; }
double cache_misses() const { return total.cache_misses() / iterations; }
#endif
};
struct event_collector {
@@ -135,16 +117,11 @@ struct event_collector {
#if defined(__linux__)
LinuxEvents<PERF_TYPE_HARDWARE> linux_events;
event_collector(bool quiet = false) : linux_events(vector<int>{
#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
PERF_COUNT_HW_CPU_CYCLES,
PERF_COUNT_HW_INSTRUCTIONS,
#else
PERF_COUNT_HW_CPU_CYCLES,
PERF_COUNT_HW_INSTRUCTIONS,
PERF_COUNT_HW_BRANCH_MISSES,
PERF_COUNT_HW_CACHE_REFERENCES,
PERF_COUNT_HW_CACHE_MISSES
#endif
}, quiet) {}
bool has_events() {
return linux_events.is_working();
@@ -156,13 +133,13 @@ struct event_collector {
}
#endif
simdjson_inline void start() {
simdjson_really_inline void start() {
#if defined(__linux)
linux_events.start();
#endif
start_clock = steady_clock::now();
}
simdjson_inline event_count& end() {
simdjson_really_inline event_count& end() {
time_point<steady_clock> end_clock = steady_clock::now();
#if defined(__linux)
linux_events.end(count.event_counts);
+1 -1
View File
@@ -33,7 +33,7 @@ struct runner : public file_runner<I> {
struct simdjson_dom;
template<typename I> simdjson_inline static void find_tweet(benchmark::State &state) {
template<typename I> simdjson_really_inline static void find_tweet(benchmark::State &state) {
run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
}
+2 -3
View File
@@ -40,14 +40,13 @@ struct rapidjson : rapidjson_base {
};
BENCHMARK_TEMPLATE(find_tweet, rapidjson)->UseManualTime();
#if SIMDJSON_COMPETITION_ONDEMAND_INSITU
struct rapidjson_insitu : rapidjson_base {
bool run(simdjson::padded_string &json, uint64_t find_id, std::string_view &result) {
return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), find_id, result);
return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), find_id, result);
}
};
BENCHMARK_TEMPLATE(find_tweet, rapidjson_insitu)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_INSITU
} // namespace find_tweet
#endif // SIMDJSON_COMPETITION_RAPIDJSON
+2 -4
View File
@@ -11,14 +11,12 @@ struct sajson {
size_t ast_buffer_size{0};
size_t *ast_buffer{nullptr};
~sajson() { free(ast_buffer); }
simdjson_inline std::string_view get_string_view(const ::sajson::value &obj, std::string_view key) {
simdjson_really_inline std::string_view get_string_view(const ::sajson::value &obj, std::string_view key) {
auto val = obj.get_value_of_key({key.data(), key.length()});
if (val.get_type() != ::sajson::TYPE_STRING) { throw "field is not a string"; }
return { val.as_cstring(), val.get_string_length() };
}
simdjson_inline uint64_t get_str_uint64(const ::sajson::value &obj, std::string_view key) {
simdjson_really_inline uint64_t get_str_uint64(const ::sajson::value &obj, std::string_view key) {
// Since sajson only supports 53-bit numbers, and IDs in twitter.json can be > 53 bits, we read the corresponding id_str and parse that.
auto val = obj.get_value_of_key({key.data(), key.length()});
if (val.get_type() != ::sajson::TYPE_STRING) { throw "field not a string"; }
+2 -2
View File
@@ -40,14 +40,14 @@ struct yyjson : yyjson_base {
}
};
BENCHMARK_TEMPLATE(find_tweet, yyjson)->UseManualTime();
#if SIMDJSON_COMPETITION_ONDEMAND_INSITU
struct yyjson_insitu : yyjson_base {
bool run(simdjson::padded_string &json, uint64_t find_id, std::string_view &result) {
return yyjson_base::run(yyjson_read_opts(json.data(), json.size(), YYJSON_READ_INSITU, 0, 0), find_id, result);
}
};
BENCHMARK_TEMPLATE(find_tweet, yyjson_insitu)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_INSITU
} // namespace find_tweet
#endif // SIMDJSON_COMPETITION_YYJSON
-48
View File
@@ -1,48 +0,0 @@
#pragma once
#include "json_benchmark/file_runner.h"
namespace json2msgpack {
using namespace json_benchmark;
template <typename I> struct runner : public file_runner<I> {
std::string_view result;
std::unique_ptr<char[]> buffer;
bool setup(benchmark::State &state) {
bool isok = this->load_json(state, TWITTER_JSON);
if (isok) {
// Let us allocate a sizeable buffer.
buffer = std::unique_ptr<char[]>(new char[this->json.size() * 4 + 1024]);
}
return isok;
}
bool before_run(benchmark::State &state) {
if (!file_runner<I>::before_run(state)) {
return false;
}
// Clear the buffer.
::memset(buffer.get(), 0, this->json.size() * 4 + 1024);
return true;
}
bool run(benchmark::State &) {
return this->implementation.run(this->json, buffer.get(), result);
}
template <typename R>
bool diff(benchmark::State &state, runner<R> &reference) {
return diff_results(state, result.size(), reference.result.size(), diff_flags::NONE);
}
};
struct simdjson_ondemand;
template <typename I>
simdjson_inline static void json2msgpack(benchmark::State &state) {
run_json_benchmark<runner<I>, runner<simdjson_ondemand>>(state);
}
} // namespace json2msgpack
-117
View File
@@ -1,117 +0,0 @@
#pragma once
#ifdef SIMDJSON_COMPETITION_NLOHMANN_JSON
#include "json2msgpack.h"
namespace json2msgpack {
using namespace nlohmann;
struct nlohmann_json2msgpack {
inline std::string_view to_msgpack(const simdjson::padded_string &json,
uint8_t *buf);
private:
inline void write_double(const double d) noexcept;
inline void write_byte(const uint8_t b) noexcept;
inline void write_uint32(const uint32_t w) noexcept;
inline void write_string(const std::string& str);
inline void recursive_processor(basic_json<> element);
uint8_t *buff{};
};
std::string_view nlohmann_json2msgpack::to_msgpack(const simdjson::padded_string &json,
uint8_t *buf) {
buff = buf;
auto val = nlohmann::json::parse(json.data(), json.data() + json.size());
recursive_processor(val);
return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
}
void nlohmann_json2msgpack::write_double(const double d) noexcept {
*buff++ = 0xcb;
::memcpy(buff, &d, sizeof(d));
buff += sizeof(d);
}
void nlohmann_json2msgpack::write_byte(const uint8_t b) noexcept {
*buff = b;
buff++;
}
void nlohmann_json2msgpack::write_uint32(const uint32_t w) noexcept {
::memcpy(buff, &w, sizeof(w));
buff += sizeof(w);
}
void nlohmann_json2msgpack::write_string(const std::string & str) {
write_byte(0xdb);
write_uint32(uint32_t(str.size()));
::memcpy(buff, str.data(), str.size());
buff += str.size();
}
void nlohmann_json2msgpack::recursive_processor(json element) {
switch (element.type()) {
case nlohmann::detail::value_t::array: {
uint32_t counter = 0;
write_byte(0xdd);
std::vector<json> array = element.get<std::vector<json>>();
write_uint32(uint32_t(array.size()));
for (auto child : array) {
recursive_processor(child);
}
} break;
case nlohmann::detail::value_t::object: {
write_byte(0xdf);
std::map<std::string,json> object = element.get<std::map<std::string,json>>();
write_uint32(uint32_t(object.size()));
for (auto field : object) {
write_string(field.first);
recursive_processor(field.second);
}
} break;
case nlohmann::detail::value_t::number_integer:
case nlohmann::detail::value_t::number_unsigned:
case nlohmann::detail::value_t::number_float:
write_double(double(element));
break;
case nlohmann::detail::value_t::string:
write_string(std::string(element));
break;
case nlohmann::detail::value_t::boolean:
write_byte(0xc2 + bool(element));
break;
case nlohmann::detail::value_t::null:
write_byte(0xc0);
break;
case nlohmann::detail::value_t::discarded:
case nlohmann::detail::value_t::binary:
default:
printf("unexpected\n");
break;
}
}
struct nlohmann_json {
using StringType = std::string_view;
nlohmann_json2msgpack parser{};
bool run(simdjson::padded_string &json, char *buffer,
std::string_view &result) {
result = parser.to_msgpack(json, reinterpret_cast<uint8_t *>(buffer));
return true;
}
};
BENCHMARK_TEMPLATE(json2msgpack, nlohmann_json)->UseManualTime();
} // namespace json2msgpack
#endif // SIMDJSON_COMPETITION_NLOHMANN_JSON
-142
View File
@@ -1,142 +0,0 @@
#pragma once
#ifdef SIMDJSON_COMPETITION_RAPIDJSON
#include "json2msgpack.h"
namespace json2msgpack {
using namespace rapidjson;
template <int parseflag>
struct rapidjson2msgpack {
inline std::string_view to_msgpack(char *json, uint8_t *buf);
private:
inline void write_double(const double d) noexcept;
inline void write_byte(const uint8_t b) noexcept;
inline void write_uint32(const uint32_t w) noexcept;
inline void write_uint32_at(const uint32_t w, uint8_t *p) noexcept;
void write_string(const char * s, size_t length) noexcept;
inline void recursive_processor(Value &v);
uint8_t *buff{};
};
template <int parseflag>
std::string_view rapidjson2msgpack<parseflag>::to_msgpack(char *json, uint8_t *buf) {
buff = buf;
Document doc{};
if(parseflag & kParseInsituFlag) {
doc.ParseInsitu<parseflag>(json);
} else {
doc.Parse<parseflag>(json);
}
recursive_processor(doc);
return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
}
template <int parseflag>
void rapidjson2msgpack<parseflag>::write_double(const double d) noexcept {
*buff++ = 0xcb;
::memcpy(buff, &d, sizeof(d));
buff += sizeof(d);
}
template <int parseflag>
void rapidjson2msgpack<parseflag>::write_byte(const uint8_t b) noexcept {
*buff = b;
buff++;
}
template <int parseflag>
void rapidjson2msgpack<parseflag>::write_string(const char * c, size_t len) noexcept {
write_byte(0xdb);
write_uint32(uint32_t(len));
::memcpy(buff, c, len);
buff += len;
}
template <int parseflag>
void rapidjson2msgpack<parseflag>::write_uint32(const uint32_t w) noexcept {
::memcpy(buff, &w, sizeof(w));
buff += sizeof(w);
}
template <int parseflag>
void rapidjson2msgpack<parseflag>::write_uint32_at(const uint32_t w, uint8_t *p) noexcept {
::memcpy(p, &w, sizeof(w));
}
template <int parseflag>
void rapidjson2msgpack<parseflag>::recursive_processor(Value &v) {
switch (v.GetType()) {
case kArrayType:
write_byte(0xdd);
write_uint32(v.Size());
for (Value::ValueIterator i = v.Begin(); i != v.End(); ++i) {
recursive_processor(*i);
}
break;
case kObjectType:
write_byte(0xdf);
write_uint32(uint32_t(v.MemberEnd()-v.MemberBegin()));
for (Value::MemberIterator m = v.MemberBegin(); m != v.MemberEnd();
++m) {
write_string(m->name.GetString(), m->name.GetStringLength());
recursive_processor(m->value);
}
break;
case kStringType:
write_string(v.GetString(), v.GetStringLength());
break;
case kNumberType:
write_double(v.GetDouble());
break;
case kFalseType:
write_byte(0xc2);
break;
case kTrueType:
write_byte(0xc3);
break;
case kNullType:
write_byte(0xc0);
break;
}
}
template <int parseflag>
struct rapidjson_base {
using StringType = std::string_view;
rapidjson2msgpack<parseflag> parser{};
bool run(simdjson::padded_string &json, char *buffer,
std::string_view &result) {
result =
parser.to_msgpack(json.data(), reinterpret_cast<uint8_t *>(buffer));
return true;
}
};
using rapidjson = rapidjson_base<kParseValidateEncodingFlag|kParseFullPrecisionFlag>;
BENCHMARK_TEMPLATE(json2msgpack, rapidjson)->UseManualTime();
#if SIMDJSON_COMPETITION_ONDEMAND_APPROX
using rapidjson_approx = rapidjson_base<kParseValidateEncodingFlag>;
BENCHMARK_TEMPLATE(json2msgpack, rapidjson_approx)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_APPROX
#if SIMDJSON_COMPETITION_ONDEMAND_INSITU
using rapidjson_insitu = rapidjson_base<kParseValidateEncodingFlag|kParseInsituFlag>;
BENCHMARK_TEMPLATE(json2msgpack, rapidjson_insitu)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_INSITU
} // namespace json2msgpack
#endif // SIMDJSON_COMPETITION_RAPIDJSON
-131
View File
@@ -1,131 +0,0 @@
#pragma once
#ifdef SIMDJSON_COMPETITION_SAJSON
#include "json2msgpack.h"
namespace json2msgpack {
using namespace sajson;
struct sajson2msgpack {
inline std::string_view to_msgpack(char *json, size_t size, uint8_t *buf);
virtual ~sajson2msgpack() { free(ast_buffer); }
private:
inline void write_double(const double d) noexcept;
inline void write_byte(const uint8_t b) noexcept;
inline void write_uint32(const uint32_t w) noexcept;
inline void write_string(const char * s, size_t length) noexcept;
inline void recursive_processor(const sajson::value &v);
uint8_t *buff{};
size_t ast_buffer_size{0};
size_t *ast_buffer{nullptr};
};
std::string_view sajson2msgpack::to_msgpack(char *json, size_t size, uint8_t *buf) {
buff = buf;
if (!ast_buffer) {
ast_buffer_size = size;
ast_buffer = (size_t *)std::malloc(ast_buffer_size * sizeof(size_t));
}
auto doc = parse(
bounded_allocation(ast_buffer, ast_buffer_size),
mutable_string_view(size, json)
);
auto root = doc.get_root();
recursive_processor(root);
return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
}
void sajson2msgpack::write_string(const char * c, size_t len) noexcept {
write_byte(0xdb);
write_uint32(uint32_t(len));
::memcpy(buff, c, len);
buff += len;
}
void sajson2msgpack::write_double(const double d) noexcept {
*buff++ = 0xcb;
::memcpy(buff, &d, sizeof(d));
buff += sizeof(d);
}
void sajson2msgpack::write_byte(const uint8_t b) noexcept {
*buff = b;
buff++;
}
void sajson2msgpack::write_uint32(const uint32_t w) noexcept {
::memcpy(buff, &w, sizeof(w));
buff += sizeof(w);
}
void sajson2msgpack::recursive_processor(const sajson::value &node) {
using namespace sajson;
switch (node.get_type()) {
case TYPE_NULL:
write_byte(0xc0);
break;
case TYPE_FALSE:
write_byte(0xc2);
break;
case TYPE_TRUE:
write_byte(0xc3);
break;
case TYPE_ARRAY: {
auto length = node.get_length();
write_byte(0xdf);
write_uint32(uint32_t(length));
for (size_t i = 0; i < length; ++i) {
recursive_processor(node.get_array_element(i));
}
break;
}
case TYPE_OBJECT: {
auto length = node.get_length();
write_byte(0xdd);
write_uint32(uint32_t(length));
for (auto i = 0u; i < length; ++i) {
auto s = node.get_object_key(i);
write_string(s.data(), s.length());
recursive_processor(node.get_object_value(i));
}
break;
}
case TYPE_STRING:
write_string(node.as_cstring(), node.get_string_length());
break;
case TYPE_DOUBLE:
case TYPE_INTEGER:
write_double(node.get_number_value());
break;
default:
assert(false && "unknown node type");
}
}
struct sajson {
using StringType = std::string_view;
sajson2msgpack parser{};
bool run(simdjson::padded_string &json, char *buffer,
std::string_view &result) {
result =
parser.to_msgpack(json.data(), json.size(), reinterpret_cast<uint8_t *>(buffer));
return true;
}
};
BENCHMARK_TEMPLATE(json2msgpack, sajson)->UseManualTime();
} // namespace json2msgpack
#endif // SIMDJSON_COMPETITION_SAJSON
-138
View File
@@ -1,138 +0,0 @@
#pragma once
#if SIMDJSON_EXCEPTIONS
#include "json2msgpack.h"
namespace json2msgpack {
using namespace simdjson;
struct simdjsondom2msgpack {
/**
* @brief Converts the provided JSON into msgpack.
*
* @param json JSON input
* @param buf temporary buffer (must be large enough, with simdjson::SIMDJSON_PADDING bytes
* of padding)
* @return std::string_view msgpack output, writting to the temporary buffer
*/
inline std::string_view to_msgpack(const simdjson::padded_string &json,
uint8_t *buf);
private:
simdjson_really_inline void write_double(const double d) noexcept;
simdjson_really_inline void write_string(const std::string_view v) noexcept;
simdjson_really_inline void write_byte(const uint8_t b) noexcept;
simdjson_really_inline void write_uint32(const uint32_t w) noexcept;
simdjson_really_inline uint8_t *skip_uint32() noexcept;
simdjson_really_inline void write_uint32_at(const uint32_t w,
uint8_t *p) noexcept;
inline void recursive_processor(simdjson::dom::element element);
dom::parser parser;
uint8_t *buff{};
};
std::string_view
simdjsondom2msgpack::to_msgpack(const simdjson::padded_string &json,
uint8_t *buf) {
buff = buf;
recursive_processor(parser.parse(json));
return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
}
void simdjsondom2msgpack::write_string(const std::string_view v) noexcept {
write_byte(0xdb);
write_uint32(uint32_t(v.size()));
::memcpy(buff, v.data(), v.size());
buff += v.size();
}
void simdjsondom2msgpack::write_double(const double d) noexcept {
*buff++ = 0xcb;
::memcpy(buff, &d, sizeof(d));
buff += sizeof(d);
}
void simdjsondom2msgpack::write_byte(const uint8_t b) noexcept {
*buff = b;
buff++;
}
void simdjsondom2msgpack::write_uint32(const uint32_t w) noexcept {
::memcpy(buff, &w, sizeof(w));
buff += sizeof(w);
}
uint8_t *simdjsondom2msgpack::skip_uint32() noexcept {
uint8_t *ret = buff;
buff += sizeof(uint32_t);
return ret;
}
void simdjsondom2msgpack::write_uint32_at(const uint32_t w, uint8_t *p) noexcept {
::memcpy(p, &w, sizeof(w));
}
void simdjsondom2msgpack::recursive_processor(simdjson::dom::element element) {
switch (element.type()) {
case dom::element_type::ARRAY: {
uint32_t counter = 0;
write_byte(0xdd);
uint8_t *location = skip_uint32();
for (auto child : dom::array(element)) {
counter++;
recursive_processor(child);
}
write_uint32_at(counter, location);}
break;
case dom::element_type::OBJECT:{
uint32_t counter = 0;
write_byte(0xdf);
uint8_t *location = skip_uint32();
for (dom::key_value_pair field : dom::object(element)) {
counter++;
write_string(field.key);
recursive_processor(field.value);
}
write_uint32_at(counter, location);
}
break;
case dom::element_type::INT64:
case dom::element_type::UINT64:
case dom::element_type::DOUBLE:
write_double( double(element));
break;
case dom::element_type::STRING:
write_string(std::string_view(element));
break;
case dom::element_type::BOOL:
write_byte(0xc2 + bool(element));
break;
case dom::element_type::NULL_VALUE:
write_byte(0xc0);
break;
default:
break;
}
}
struct simdjson_dom {
using StringType = std::string_view;
simdjsondom2msgpack parser{};
bool run(simdjson::padded_string &json, char *buffer,
std::string_view &result) {
result = parser.to_msgpack(json, reinterpret_cast<uint8_t *>(buffer));
return true;
}
};
BENCHMARK_TEMPLATE(json2msgpack, simdjson_dom)->UseManualTime();
} // namespace json2msgpack
#endif // SIMDJSON_EXCEPTIONS
-251
View File
@@ -1,251 +0,0 @@
#pragma once
#if SIMDJSON_EXCEPTIONS
#include "json2msgpack.h"
namespace json2msgpack {
using namespace simdjson;
/**
* @brief The simdjson2msgpack struct is used to quickly convert
* JSON strings to msgpack views. You must provide a pointer to
* a large memory region where the msgpack gets written. The
* buffer should be large enough to store the msgpack output (which
* can never be 3x larger than the input JSON) with an additional
* simdjson::SIMDJSON_PADDING bytes.
*
* Recommended usage:
*
* simdjson2msgpack parser{};
* simdjson::padded_string json = "[1,2]"_padded; // some JSON
* uint8_t * buffer = new uint8_t[3*json.size() + simdjson::SIMDJSON_PADDING]; // large buffer
*
* std::string_view msgpack = parser.to_msgpack(json, buffer);
*
* The result (msgpack) is a string view to a msgpack serialization of the input JSON,
* it points inside the buffer you provided.
*
* You may reuse the simdjson2msgpack instance though you should use
* one per thread.
*/
struct simdjson2msgpack {
/**
* @brief Converts the provided JSON into msgpack.
*
* @param json JSON input
* @param buf temporary buffer (must be large enough, with simdjson::SIMDJSON_PADDING bytes
* of padding)
* @return std::string_view msgpack output, writting to the temporary buffer
*/
inline std::string_view to_msgpack(const simdjson::padded_string &json,
uint8_t *buf);
private:
simdjson_inline void write_double(const double d) noexcept;
simdjson_inline void write_byte(const uint8_t b) noexcept;
simdjson_inline void write_uint32(const uint32_t w) noexcept;
simdjson_inline uint8_t *skip_uint32() noexcept;
simdjson_inline void write_uint32_at(const uint32_t w,
uint8_t *p) noexcept;
simdjson_inline void
write_raw_string(simdjson::ondemand::raw_json_string rjs);
inline void recursive_processor(simdjson::ondemand::value element);
inline void recursive_processor_ref(simdjson::ondemand::value& element);
simdjson::ondemand::parser parser;
uint8_t *buff{};
};
std::string_view
simdjson2msgpack::to_msgpack(const simdjson::padded_string &json,
uint8_t *buf) {
buff = buf;
ondemand::document doc = parser.iterate(json);
if (doc.is_scalar()) {
// we have a special case where the JSON document is a single document...
switch (doc.type()) {
case simdjson::ondemand::json_type::number:
write_double(doc.get_double());
break;
case simdjson::ondemand::json_type::string:
write_raw_string(doc.get_raw_json_string());
break;
case simdjson::ondemand::json_type::boolean:
write_byte(0xc2 + doc.get_bool());
break;
case simdjson::ondemand::json_type::null:
// We check that the value is indeed null
// otherwise: an error is thrown.
if(doc.is_null()) {
write_byte(0xc0);
}
break;
case simdjson::ondemand::json_type::array:
case simdjson::ondemand::json_type::object:
default:
// impossible
SIMDJSON_UNREACHABLE();
}
} else {
simdjson::ondemand::value val = doc;
#define SIMDJSON_GCC_COMPILER ((__GNUC__) && !(__clang__) && !(__INTEL_COMPILER))
#if SIMDJSON_GCC_COMPILER
// the GCC compiler does well with by-value passing.
// GCC has superior recursive inlining:
// https://stackoverflow.com/questions/29186186/why-does-gcc-generate-a-faster-program-than-clang-in-this-recursive-fibonacci-co
// https://godbolt.org/z/TeK4doE51
recursive_processor(val);
#else
recursive_processor_ref(val);
#endif
}
if (doc.current_location().error() == simdjson::SUCCESS) {
// Example of error detection - this won't be reached on twitter.json in the benchmark.
throw "There are unexpectedly tokens after the end of the json in the json2msgpack sample data";
}
return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
}
void simdjson2msgpack::write_double(const double d) noexcept {
*buff++ = 0xcb;
::memcpy(buff, &d, sizeof(d));
buff += sizeof(d);
}
void simdjson2msgpack::write_byte(const uint8_t b) noexcept {
*buff = b;
buff++;
}
void simdjson2msgpack::write_uint32(const uint32_t w) noexcept {
::memcpy(buff, &w, sizeof(w));
buff += sizeof(w);
}
uint8_t *simdjson2msgpack::skip_uint32() noexcept {
uint8_t *ret = buff;
buff += sizeof(uint32_t);
return ret;
}
void simdjson2msgpack::write_uint32_at(const uint32_t w, uint8_t *p) noexcept {
::memcpy(p, &w, sizeof(w));
}
void simdjson2msgpack::write_raw_string(
simdjson::ondemand::raw_json_string in) {
write_byte(0xdb);
uint8_t *location = skip_uint32();
std::string_view v = parser.unescape(in, buff);
write_uint32_at(uint32_t(v.size()), location);
}
void simdjson2msgpack::recursive_processor(simdjson::ondemand::value element) {
switch (element.type()) {
case simdjson::ondemand::json_type::array: {
uint32_t counter = 0;
write_byte(0xdd);
uint8_t *location = skip_uint32();
for (auto child : element.get_array()) {
counter++;
recursive_processor(child.value());
}
write_uint32_at(counter, location);
} break;
case simdjson::ondemand::json_type::object: {
uint32_t counter = 0;
write_byte(0xdf);
uint8_t *location = skip_uint32();
for (auto field : element.get_object()) {
counter++;
write_raw_string(field.key());
recursive_processor(field.value());
}
write_uint32_at(counter, location);
} break;
case simdjson::ondemand::json_type::number:
write_double(element.get_double());
break;
case simdjson::ondemand::json_type::string:
write_raw_string(element.get_raw_json_string());
break;
case simdjson::ondemand::json_type::boolean:
write_byte(0xc2 + element.get_bool());
break;
case simdjson::ondemand::json_type::null:
// We check that the value is indeed null
// otherwise: an error is thrown.
if(element.is_null()) {
write_byte(0xc0);
}
break;
default:
SIMDJSON_UNREACHABLE();
}
}
void simdjson2msgpack::recursive_processor_ref(simdjson::ondemand::value& element) {
switch (element.type()) {
case simdjson::ondemand::json_type::array: {
uint32_t counter = 0;
write_byte(0xdd);
uint8_t *location = skip_uint32();
for (auto child : element.get_array()) {
counter++;
simdjson::ondemand::value v = child.value();
recursive_processor_ref(v);
}
write_uint32_at(counter, location);
} break;
case simdjson::ondemand::json_type::object: {
uint32_t counter = 0;
write_byte(0xdf);
uint8_t *location = skip_uint32();
for (auto field : element.get_object()) {
counter++;
write_raw_string(field.key());
simdjson::ondemand::value v = field.value();
recursive_processor_ref(v);
}
write_uint32_at(counter, location);
} break;
case simdjson::ondemand::json_type::number:
write_double(element.get_double());
break;
case simdjson::ondemand::json_type::string:
write_raw_string(element.get_raw_json_string());
break;
case simdjson::ondemand::json_type::boolean:
write_byte(0xc2 + element.get_bool());
break;
case simdjson::ondemand::json_type::null:
// We check that the value is indeed null
// otherwise: an error is thrown.
if(element.is_null()) {
write_byte(0xc0);
}
break;
default:
SIMDJSON_UNREACHABLE();
}
}
struct simdjson_ondemand {
using StringType = std::string_view;
simdjson2msgpack parser{};
bool run(simdjson::padded_string &json, char *buffer,
std::string_view &result) {
result = parser.to_msgpack(json, reinterpret_cast<uint8_t *>(buffer));
return true;
}
};
BENCHMARK_TEMPLATE(json2msgpack, simdjson_ondemand)->UseManualTime();
} // namespace json2msgpack
#endif // SIMDJSON_EXCEPTIONS
-123
View File
@@ -1,123 +0,0 @@
#pragma once
#ifdef SIMDJSON_COMPETITION_YYJSON
#include "json2msgpack.h"
namespace json2msgpack {
struct yyjson2msgpack {
inline std::string_view to_msgpack(yyjson_doc *doc, uint8_t *buf);
private:
inline void write_double(const double d) noexcept;
inline void write_byte(const uint8_t b) noexcept;
inline void write_uint32(const uint32_t w) noexcept;
inline void write_string(const char *s, size_t length) noexcept;
inline void recursive_processor(yyjson_val *obj);
uint8_t *buff{};
};
std::string_view yyjson2msgpack::to_msgpack(yyjson_doc *doc, uint8_t *buf) {
buff = buf;
yyjson_val *root = yyjson_doc_get_root(doc);
recursive_processor(root);
return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
}
void yyjson2msgpack::write_string(const char *c, size_t len) noexcept {
write_byte(0xdb);
write_uint32(uint32_t(len));
::memcpy(buff, c, len);
buff += len;
}
void yyjson2msgpack::write_double(const double d) noexcept {
*buff++ = 0xcb;
::memcpy(buff, &d, sizeof(d));
buff += sizeof(d);
}
void yyjson2msgpack::write_byte(const uint8_t b) noexcept {
*buff = b;
buff++;
}
void yyjson2msgpack::write_uint32(const uint32_t w) noexcept {
::memcpy(buff, &w, sizeof(w));
buff += sizeof(w);
}
void yyjson2msgpack::recursive_processor(yyjson_val *obj) {
size_t idx, max;
yyjson_val *val;
yyjson_val *key;
switch (yyjson_get_type(obj)) {
case YYJSON_TYPE_STR:
write_string(yyjson_get_str(obj), yyjson_get_len(obj));
break;
case YYJSON_TYPE_ARR:
write_byte(0xdf);
write_uint32(uint32_t(yyjson_arr_size(obj)));
yyjson_arr_foreach(obj, idx, max, val) { recursive_processor(val); }
break;
case YYJSON_TYPE_OBJ:
write_byte(0xdd);
write_uint32(uint32_t(yyjson_obj_size(obj)));
yyjson_obj_foreach(obj, idx, max, key, val) {
write_string(yyjson_get_str(key), yyjson_get_len(key));
recursive_processor(val);
}
break;
case YYJSON_TYPE_BOOL:
write_byte(0xc2 + yyjson_get_bool(obj));
break;
case YYJSON_TYPE_NULL:
write_byte(0xc0);
break;
case YYJSON_TYPE_NUM:
switch (yyjson_get_subtype(obj)) {
case YYJSON_SUBTYPE_UINT:
write_double(double(yyjson_get_uint(obj)));
break;
case YYJSON_SUBTYPE_SINT:
write_double(double(yyjson_get_sint(obj)));
break;
case YYJSON_SUBTYPE_REAL:
write_double(yyjson_get_real(obj));
break;
default:
SIMDJSON_UNREACHABLE();
}
break;
default:
SIMDJSON_UNREACHABLE();
}
}
struct yyjson : yyjson2msgpack {
bool run(simdjson::padded_string &json, char *buffer,
std::string_view &result) {
yyjson_doc *doc = yyjson_read(json.data(), json.size(), 0);
result = to_msgpack(doc, reinterpret_cast<uint8_t*>(buffer));
return true;
}
};
BENCHMARK_TEMPLATE(json2msgpack, yyjson)->UseManualTime();
#if SIMDJSON_COMPETITION_ONDEMAND_INSITU
struct yyjson_insitu : yyjson2msgpack {
bool run(simdjson::padded_string &json, char *buffer,
std::string_view &result) {
yyjson_doc *doc =
yyjson_read_opts(json.data(), json.size(), YYJSON_READ_INSITU, 0, 0);
result = to_msgpack(doc, reinterpret_cast<uint8_t*>(buffer));
return true;
}
};
BENCHMARK_TEMPLATE(json2msgpack, yyjson_insitu)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_INSITU
} // namespace json2msgpack
#endif // SIMDJSON_COMPETITION_YYJSON
@@ -10,8 +10,8 @@ void maybe_display_implementation() {
static bool displayed_implementation = false;
if(!displayed_implementation) {
displayed_implementation = true;
std::cout << "simdjson::dom implementation: " << simdjson::get_active_implementation()->name() << std::endl;
std::cout << "simdjson::ondemand implementation (stage 1): " << simdjson::get_active_implementation()->name() << std::endl;
std::cout << "simdjson::dom implementation: " << simdjson::active_implementation->name() << std::endl;
std::cout << "simdjson::ondemand implementation (stage 1): " << simdjson::active_implementation->name() << std::endl;
std::cout << "simdjson::ondemand implementation (stage 2): " << simdjson::builtin_implementation()->name() << std::endl;
}
}
@@ -58,11 +58,10 @@ template<typename B, typename R> static void run_json_benchmark(benchmark::State
if (collector.has_events()) {
state.counters["instructions"] = events.instructions();
state.counters["cycles"] = events.cycles();
#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
state.counters["branch_miss"] = events.branch_misses();
state.counters["cache_miss"] = events.cache_misses();
state.counters["cache_ref"] = events.cache_references();
#endif
state.counters["instructions_per_byte"] = events.instructions() / double(bench.bytes_per_iteration());
state.counters["instructions_per_cycle"] = events.instructions() / events.cycles();
state.counters["cycles_per_byte"] = events.cycles() / double(bench.bytes_per_iteration());
@@ -70,11 +69,9 @@ template<typename B, typename R> static void run_json_benchmark(benchmark::State
state.counters["best_instructions"] = events.best.instructions();
state.counters["best_cycles"] = events.best.cycles();
#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
state.counters["best_branch_miss"] = events.best.branch_misses();
state.counters["best_cache_miss"] = events.best.cache_misses();
state.counters["best_cache_ref"] = events.best.cache_references();
#endif
state.counters["best_instructions_per_byte"] = events.best.instructions() / double(bench.bytes_per_iteration());
state.counters["best_instructions_per_cycle"] = events.best.instructions() / events.best.cycles();
@@ -95,11 +92,9 @@ template<typename B, typename R> static void run_json_benchmark(benchmark::State
if (collector.has_events()) {
label << " instructions=" << setw(12) << uint64_t(events.best.instructions()) << setw(0);
label << " cycles=" << setw(12) << uint64_t(events.best.cycles()) << setw(0);
#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
label << " branch_miss=" << setw(8) << uint64_t(events.best.branch_misses()) << setw(0);
label << " cache_miss=" << setw(8) << uint64_t(events.best.cache_misses()) << setw(0);
label << " cache_ref=" << setw(10) << uint64_t(events.best.cache_references()) << setw(0);
#endif
}
label << " items=" << setw(10) << bench.items_per_iteration() << setw(0);
+1 -1
View File
@@ -79,7 +79,7 @@ static const simdjson::padded_string &get_built_json_array() {
struct simdjson_dom;
template<typename I> simdjson_inline static void kostya(benchmark::State &state) {
template<typename I> simdjson_really_inline static void kostya(benchmark::State &state) {
run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
}
+1 -1
View File
@@ -42,7 +42,7 @@ struct nlohmann_json_sax {
return true;
}
bool number_unsigned(number_unsigned_t val) override { // Need this event because coordinate value can be equal to 1
buffer[k] = double(val);
buffer[k] = val;
if (k == 2) {
result.emplace_back(json_benchmark::point{buffer[0],buffer[1],buffer[2]});
k = 0;
+9 -10
View File
@@ -13,7 +13,7 @@ struct rapidjson_base {
Document doc;
simdjson_inline double get_double(Value &object, std::string_view key) {
simdjson_really_inline double get_double(Value &object, std::string_view key) {
auto field = object.FindMember(key.data());
if (field == object.MemberEnd()) { throw "Missing double field"; }
if (!field->value.IsNumber()) { throw "Field is not double"; }
@@ -34,29 +34,28 @@ struct rapidjson_base {
return true;
}
};
#if SIMDJSON_COMPETITION_ONDEMAND_APPROX
struct rapidjson_approx : rapidjson_base {
struct rapidjson : rapidjson_base {
bool run(simdjson::padded_string &json, std::vector<point> &result) {
return rapidjson_base::run(doc.Parse<kParseValidateEncodingFlag>(json.data()), result);
}
};
BENCHMARK_TEMPLATE(kostya, rapidjson_approx)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_APPROX
BENCHMARK_TEMPLATE(kostya, rapidjson)->UseManualTime();
struct rapidjson : rapidjson_base {
struct rapidjson_lossless : rapidjson_base {
bool run(simdjson::padded_string &json, std::vector<point> &result) {
return rapidjson_base::run(doc.Parse<kParseValidateEncodingFlag | kParseFullPrecisionFlag>(json.data()), result);
}
};
BENCHMARK_TEMPLATE(kostya, rapidjson)->UseManualTime();
#if SIMDJSON_COMPETITION_ONDEMAND_INSITU
BENCHMARK_TEMPLATE(kostya, rapidjson_lossless)->UseManualTime();
struct rapidjson_insitu : rapidjson_base {
bool run(simdjson::padded_string &json, std::vector<point> &result) {
return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), result);
return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), result);
}
};
BENCHMARK_TEMPLATE(kostya, rapidjson_insitu)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_INSITU
} // namespace kostya
#endif // SIMDJSON_COMPETITION_RAPIDJSON
+1 -2
View File
@@ -11,9 +11,8 @@ struct sajson {
size_t ast_buffer_size{0};
size_t *ast_buffer{nullptr};
~sajson() { free(ast_buffer); }
simdjson_inline double get_double(const ::sajson::value &obj, std::string_view key) {
simdjson_really_inline double get_double(const ::sajson::value &obj, std::string_view key) {
using namespace sajson;
auto val = obj.get_value_of_key({key.data(), key.length()});
+3 -3
View File
@@ -9,7 +9,7 @@ namespace kostya {
struct yyjson_base {
static constexpr diff_flags DiffFlags = diff_flags::NONE;
simdjson_inline double get_double(yyjson_val *obj, std::string_view key) {
simdjson_really_inline double get_double(yyjson_val *obj, std::string_view key) {
yyjson_val *val = yyjson_obj_getn(obj, key.data(), key.length());
if (!val) { throw "missing point field!"; }
if (yyjson_get_type(val) != YYJSON_TYPE_NUM) { throw "Number is not a type!"; }
@@ -53,14 +53,14 @@ struct yyjson : yyjson_base {
}
};
BENCHMARK_TEMPLATE(kostya, yyjson)->UseManualTime();
#if SIMDJSON_COMPETITION_ONDEMAND_INSITU
struct yyjson_insitu : yyjson_base {
bool run(simdjson::padded_string &json, std::vector<point> &result) {
return yyjson_base::run(yyjson_read_opts(json.data(), json.size(), YYJSON_READ_INSITU, 0, 0), result);
}
};
BENCHMARK_TEMPLATE(kostya, yyjson_insitu)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_INSITU
} // namespace kostya
#endif // SIMDJSON_COMPETITION_YYJSON
@@ -6,9 +6,6 @@
namespace large_amazon_cellphones {
const bool UNTHREADED = false;
const bool THREADED = true;
static const simdjson::padded_string &get_built_json();
using namespace json_benchmark;
@@ -16,11 +13,11 @@ using namespace json_benchmark;
struct brand {
double cumulative_rating;
uint64_t reviews_count;
simdjson_inline bool operator==(const brand &other) const {
simdjson_really_inline bool operator==(const brand &other) const {
return cumulative_rating == other.cumulative_rating &&
reviews_count == other.reviews_count;
}
simdjson_inline bool operator!=(const brand &other) const { return !(*this == other); }
simdjson_really_inline bool operator!=(const brand &other) const { return !(*this == other); }
};
simdjson_unused static std::ostream &operator<<(std::ostream &o, const brand &b) {
@@ -84,11 +81,11 @@ static const simdjson::padded_string &get_built_json() {
return json;
}
template<bool threaded>
struct simdjson_dom;
template<typename I> simdjson_inline static void large_amazon_cellphones(benchmark::State &state) {
run_json_benchmark<runner<I>, runner<simdjson_dom<UNTHREADED>>>(state);
template<typename I> simdjson_really_inline static void large_amazon_cellphones(benchmark::State &state) {
run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
}
} // namespace large_amazon_cellphones
} // namespace large_amazon_cellphones
@@ -9,16 +9,12 @@ namespace large_amazon_cellphones {
using namespace simdjson;
template<bool threaded>
struct simdjson_dom {
using StringType = std::string;
dom::parser parser{};
bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
#ifdef SIMDJSON_THREADS_ENABLED
parser.threaded = threaded;
#endif
auto stream = parser.parse_many(json);
auto i = stream.begin();
++i; // Skip first line
@@ -42,10 +38,7 @@ struct simdjson_dom {
};
BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_dom<UNTHREADED>)->UseManualTime();
#ifdef SIMDJSON_THREADS_ENABLED
BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_dom<THREADED>)->UseManualTime();
#endif
BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_dom)->UseManualTime();
} // namespace large_amazon_cellphones
@@ -8,21 +8,17 @@ namespace large_amazon_cellphones {
using namespace simdjson;
template<bool threaded>
struct simdjson_ondemand {
using StringType = std::string;
ondemand::parser parser{};
bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
#ifdef SIMDJSON_THREADS_ENABLED
parser.threaded = threaded;
#endif
ondemand::document_stream stream = parser.iterate_many(json);
ondemand::document_stream::iterator i = stream.begin();
++i; // Skip first line
for (;i != stream.end(); ++i) {
auto doc = *i;
auto & doc = *i;
size_t index{0};
StringType copy;
double rating;
@@ -62,10 +58,7 @@ struct simdjson_ondemand {
};
BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_ondemand<UNTHREADED>)->UseManualTime();
#ifdef SIMDJSON_THREADS_ENABLED
BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_ondemand<THREADED>)->UseManualTime();
#endif
BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_ondemand)->UseManualTime();
} // namespace amazon_cellphones
+1 -1
View File
@@ -34,7 +34,7 @@ struct nlohmann_json_sax {
return true;
}
bool number_unsigned(number_unsigned_t val) override {
buffer[k] = double(val);
buffer[k] = val;
if (k == 2) {
result.emplace_back(json_benchmark::point{buffer[0],buffer[1],buffer[2]});
k = 0;
+8 -10
View File
@@ -13,7 +13,7 @@ struct rapidjson_base {
Document doc;
simdjson_inline double get_double(Value &object, std::string_view key) {
simdjson_really_inline double get_double(Value &object, std::string_view key) {
auto field = object.FindMember(key.data());
if (field == object.MemberEnd()) { throw "Missing double field"; }
if (!field->value.IsNumber()) { throw "Field is not double"; }
@@ -31,30 +31,28 @@ struct rapidjson_base {
return true;
}
};
#if SIMDJSON_COMPETITION_ONDEMAND_APPROX
struct rapidjson_approx : rapidjson_base {
struct rapidjson : rapidjson_base {
bool run(simdjson::padded_string &json, std::vector<point> &result) {
return rapidjson_base::run(doc.Parse<kParseValidateEncodingFlag>(json.data()), result);
}
};
BENCHMARK_TEMPLATE(large_random, rapidjson_approx)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_APPROX
BENCHMARK_TEMPLATE(large_random, rapidjson)->UseManualTime();
struct rapidjson : rapidjson_base {
struct rapidjson_lossless : rapidjson_base {
bool run(simdjson::padded_string &json, std::vector<point> &result) {
return rapidjson_base::run(doc.Parse<kParseValidateEncodingFlag | kParseFullPrecisionFlag>(json.data()), result);
}
};
BENCHMARK_TEMPLATE(large_random, rapidjson)->UseManualTime();
BENCHMARK_TEMPLATE(large_random, rapidjson_lossless)->UseManualTime();
#if SIMDJSON_COMPETITION_ONDEMAND_INSITU
struct rapidjson_insitu : rapidjson_base {
bool run(simdjson::padded_string &json, std::vector<point> &result) {
return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), result);
return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), result);
}
};
BENCHMARK_TEMPLATE(large_random, rapidjson_insitu)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_INSITU
} // namespace large_random
+1 -2
View File
@@ -11,9 +11,8 @@ struct sajson {
size_t ast_buffer_size{0};
size_t *ast_buffer{nullptr};
~sajson() { free(ast_buffer); }
simdjson_inline double get_double(const ::sajson::value &obj, std::string_view key) {
simdjson_really_inline double get_double(const ::sajson::value &obj, std::string_view key) {
using namespace sajson;
auto val = obj.get_value_of_key({key.data(), key.length()});
+3 -3
View File
@@ -9,7 +9,7 @@ namespace large_random {
struct yyjson_base {
static constexpr diff_flags DiffFlags = diff_flags::NONE;
simdjson_inline double get_double(yyjson_val *obj, std::string_view key) {
simdjson_really_inline double get_double(yyjson_val *obj, std::string_view key) {
yyjson_val *val = yyjson_obj_getn(obj, key.data(), key.length());
if (!val) { throw "missing point field!"; }
if (yyjson_get_type(val) != YYJSON_TYPE_NUM) { throw "Number is not a type!"; }
@@ -51,14 +51,14 @@ struct yyjson : yyjson_base {
}
};
BENCHMARK_TEMPLATE(large_random, yyjson)->UseManualTime();
#if SIMDJSON_COMPETITION_ONDEMAND_INSITU
struct yyjson_insitu : yyjson_base {
bool run(simdjson::padded_string &json, std::vector<point> &result) {
return yyjson_base::run(yyjson_read_opts(json.data(), json.size(), YYJSON_READ_INSITU, 0, 0), result);
}
};
BENCHMARK_TEMPLATE(large_random, yyjson_insitu)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_INSITU
} // namespace large_random
#endif // SIMDJSON_COMPETITION_YYJSON
+6 -6
View File
@@ -10,28 +10,28 @@ using namespace simdjson;
class Iter {
public:
simdjson_inline bool Run(const padded_string &json);
simdjson_really_inline bool Run(const padded_string &json);
simdjson_inline const std::vector<my_point> &Result() { return container; }
simdjson_inline size_t ItemCount() { return container.size(); }
simdjson_really_inline const std::vector<my_point> &Result() { return container; }
simdjson_really_inline size_t ItemCount() { return container.size(); }
private:
ondemand::parser parser{};
std::vector<my_point> container{};
simdjson_inline double first_double(ondemand::json_iterator &iter) {
simdjson_really_inline double first_double(ondemand::json_iterator &iter) {
if (iter.start_object().error() || iter.field_key().error() || iter.field_value()) { throw "Invalid field"; }
return iter.consume_double();
}
simdjson_inline double next_double(ondemand::json_iterator &iter) {
simdjson_really_inline double next_double(ondemand::json_iterator &iter) {
if (!iter.has_next_field() || iter.field_key().error() || iter.field_value()) { throw "Invalid field"; }
return iter.consume_double();
}
};
simdjson_inline bool Iter::Run(const padded_string &json) {
simdjson_really_inline bool Iter::Run(const padded_string &json) {
container.clear();
auto iter = parser.iterate_raw(json).value();
+4 -4
View File
@@ -10,16 +10,16 @@ using namespace simdjson;
class OnDemand {
public:
simdjson_inline bool Run(const padded_string &json);
simdjson_inline const std::vector<my_point> &Result() { return container; }
simdjson_inline size_t ItemCount() { return container.size(); }
simdjson_really_inline bool Run(const padded_string &json);
simdjson_really_inline const std::vector<my_point> &Result() { return container; }
simdjson_really_inline size_t ItemCount() { return container.size(); }
private:
ondemand::parser parser{};
std::vector<my_point> container{};
};
simdjson_inline bool OnDemand::Run(const padded_string &json) {
simdjson_really_inline bool OnDemand::Run(const padded_string &json) {
container.clear();
auto doc = parser.iterate(json);
+16 -16
View File
@@ -11,13 +11,13 @@ using namespace simdjson::builtin::stage2;
class Sax {
public:
simdjson_inline bool Run(const padded_string &json) noexcept;
simdjson_really_inline bool Run(const padded_string &json) noexcept;
simdjson_inline const std::vector<my_point> &Result() { return container; }
simdjson_inline size_t ItemCount() { return container.size(); }
simdjson_really_inline const std::vector<my_point> &Result() { return container; }
simdjson_really_inline size_t ItemCount() { return container.size(); }
private:
simdjson_inline error_code RunNoExcept(const padded_string &json) noexcept;
simdjson_really_inline error_code RunNoExcept(const padded_string &json) noexcept;
error_code Allocate(size_t new_capacity);
std::unique_ptr<uint8_t[]> string_buf{};
size_t capacity{};
@@ -34,21 +34,21 @@ public:
explicit sax_point_reader_visitor(std::vector<my_point> &_points) : points(_points) {}
simdjson_inline error_code visit_object_start(json_iterator &) {
simdjson_really_inline error_code visit_object_start(json_iterator &) {
idx = 0;
return SUCCESS;
}
simdjson_inline error_code visit_primitive(json_iterator &, const uint8_t *value) {
simdjson_really_inline error_code visit_primitive(json_iterator &, const uint8_t *value) {
if(idx == GOT_SOMETHING_ELSE) { return simdjson::SUCCESS; }
return numberparsing::parse_double(value).get(buffer[idx]);
}
simdjson_inline error_code visit_object_end(json_iterator &) {
simdjson_really_inline error_code visit_object_end(json_iterator &) {
points.emplace_back(my_point{buffer[0], buffer[1], buffer[2]});
return SUCCESS;
}
simdjson_inline error_code visit_document_start(json_iterator &) { return SUCCESS; }
simdjson_inline error_code visit_key(json_iterator &, const uint8_t * key) {
simdjson_really_inline error_code visit_document_start(json_iterator &) { return SUCCESS; }
simdjson_really_inline error_code visit_key(json_iterator &, const uint8_t * key) {
switch(key[1]) {
// Technically, we should check the other characters
// in the key, but we are cheating to go as fast
@@ -67,13 +67,13 @@ public:
}
return SUCCESS;
}
simdjson_inline error_code visit_array_start(json_iterator &) { return SUCCESS; }
simdjson_inline error_code visit_array_end(json_iterator &) { return SUCCESS; }
simdjson_inline error_code visit_document_end(json_iterator &) { return SUCCESS; }
simdjson_inline error_code visit_empty_array(json_iterator &) { return SUCCESS; }
simdjson_inline error_code visit_empty_object(json_iterator &) { return SUCCESS; }
simdjson_inline error_code visit_root_primitive(json_iterator &, const uint8_t *) { return SUCCESS; }
simdjson_inline error_code increment_count(json_iterator &) { return SUCCESS; }
simdjson_really_inline error_code visit_array_start(json_iterator &) { return SUCCESS; }
simdjson_really_inline error_code visit_array_end(json_iterator &) { return SUCCESS; }
simdjson_really_inline error_code visit_document_end(json_iterator &) { return SUCCESS; }
simdjson_really_inline error_code visit_empty_array(json_iterator &) { return SUCCESS; }
simdjson_really_inline error_code visit_empty_object(json_iterator &) { return SUCCESS; }
simdjson_really_inline error_code visit_root_primitive(json_iterator &, const uint8_t *) { return SUCCESS; }
simdjson_really_inline error_code increment_count(json_iterator &) { return SUCCESS; }
};
// NOTE: this assumes the dom_parser is already allocated
+2 -2
View File
@@ -9,7 +9,7 @@ namespace partial_tweets {
struct nlohmann_json {
using StringType=std::string;
simdjson_inline uint64_t nullable_int(nlohmann::json value) {
simdjson_really_inline uint64_t nullable_int(nlohmann::json value) {
if (value.is_null()) { return 0; }
return value;
}
@@ -37,4 +37,4 @@ BENCHMARK_TEMPLATE(partial_tweets, nlohmann_json)->UseManualTime();
} // namespace partial_tweets
#endif // SIMDJSON_COMPETITION_NLOHMANN_JSON
#endif // SIMDJSON_COMPETITION_NLOHMANN_JSON
+6 -6
View File
@@ -17,27 +17,27 @@ public:
displayed_implementation = true;
}
}
simdjson_inline bool Run(const padded_string &json);
simdjson_inline const std::vector<tweet> &Result() { return tweets; }
simdjson_inline size_t ItemCount() { return tweets.size(); }
simdjson_really_inline bool Run(const padded_string &json);
simdjson_really_inline const std::vector<tweet> &Result() { return tweets; }
simdjson_really_inline size_t ItemCount() { return tweets.size(); }
private:
ondemand::parser parser{};
std::vector<tweet> tweets{};
simdjson_inline uint64_t nullable_int(ondemand::value value) {
simdjson_really_inline uint64_t nullable_int(ondemand::value value) {
if (value.is_null()) { return 0; }
return value;
}
simdjson_inline twitter_user read_user(ondemand::object user) {
simdjson_really_inline twitter_user read_user(ondemand::object user) {
return { user.find_field("id"), user.find_field("screen_name") };
}
static inline bool displayed_implementation = false;
};
simdjson_inline bool OnDemand::Run(const padded_string &json) {
simdjson_really_inline bool OnDemand::Run(const padded_string &json) {
tweets.clear();
// Walk the document, parsing the tweets as we go
+1 -1
View File
@@ -39,7 +39,7 @@ struct runner : public file_runner<I> {
struct simdjson_dom;
template<typename I> simdjson_inline static void partial_tweets(benchmark::State &state) {
template<typename I> simdjson_really_inline static void partial_tweets(benchmark::State &state) {
run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
}
+7 -7
View File
@@ -13,27 +13,27 @@ struct rapidjson_base {
Document doc{};
simdjson_inline std::string_view get_string_view(Value &object, std::string_view key) {
simdjson_really_inline std::string_view get_string_view(Value &object, std::string_view key) {
// TODO use version that supports passing string length?
auto field = object.FindMember(key.data());
if (field == object.MemberEnd()) { throw "Missing object field"; }
if (!field->value.IsString()) { throw "Field is not a string"; }
return { field->value.GetString(), field->value.GetStringLength() };
}
simdjson_inline uint64_t get_uint64(Value &object, std::string_view key) {
simdjson_really_inline uint64_t get_uint64(Value &object, std::string_view key) {
auto field = object.FindMember(key.data());
if (field == object.MemberEnd()) { throw "Missing object field"; }
if (!field->value.IsUint64()) { throw "Field is not uint64"; }
return field->value.GetUint64();
}
simdjson_inline uint64_t get_nullable_uint64(Value &object, std::string_view key) {
simdjson_really_inline uint64_t get_nullable_uint64(Value &object, std::string_view key) {
auto field = object.FindMember(key.data());
if (field == object.MemberEnd()) { throw "Missing nullable uint64 field"; }
if (field->value.IsNull()) { return 0; }
if (!field->value.IsUint64()) { throw "Field is not nullable uint64"; }
return field->value.GetUint64();
}
simdjson_inline partial_tweets::twitter_user<std::string_view> get_user(Value &object, std::string_view key) {
simdjson_really_inline partial_tweets::twitter_user<std::string_view> get_user(Value &object, std::string_view key) {
auto field = object.FindMember(key.data());
if (field == object.MemberEnd()) { throw "Missing user field"; }
if (!field->value.IsObject()) { throw "User field is not an object"; }
@@ -67,14 +67,14 @@ struct rapidjson : rapidjson_base {
}
};
BENCHMARK_TEMPLATE(partial_tweets, rapidjson)->UseManualTime();
#if SIMDJSON_COMPETITION_ONDEMAND_INSITU
struct rapidjson_insitu : rapidjson_base {
bool run(simdjson::padded_string &json, std::vector<tweet<std::string_view>> &result) {
return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), result);
return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), result);
}
};
BENCHMARK_TEMPLATE(partial_tweets, rapidjson_insitu)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_INSITU
} // namespace partial_tweets
#endif // SIMDJSON_COMPETITION_RAPIDJSON
+5 -7
View File
@@ -11,14 +11,12 @@ struct sajson {
size_t ast_buffer_size{0};
size_t *ast_buffer{nullptr};
~sajson() { free(ast_buffer); }
simdjson_inline std::string_view get_string_view(const ::sajson::value &obj, std::string_view key) {
simdjson_really_inline std::string_view get_string_view(const ::sajson::value &obj, std::string_view key) {
auto val = obj.get_value_of_key({key.data(), key.length()});
if (val.get_type() != ::sajson::TYPE_STRING) { throw "field is not a string"; }
return { val.as_cstring(), val.get_string_length() };
}
simdjson_inline uint64_t get_uint52(const ::sajson::value &obj, std::string_view key) {
simdjson_really_inline uint64_t get_uint52(const ::sajson::value &obj, std::string_view key) {
auto val = obj.get_value_of_key({key.data(), key.length()});
switch (val.get_type()) {
case ::sajson::TYPE_INTEGER: {
@@ -30,7 +28,7 @@ struct sajson {
throw "field not integer";
}
}
simdjson_inline uint64_t get_str_uint64(const ::sajson::value &obj, std::string_view key) {
simdjson_really_inline uint64_t get_str_uint64(const ::sajson::value &obj, std::string_view key) {
// Since sajson only supports 53-bit numbers, and IDs in twitter.json can be > 53 bits, we read the corresponding id_str and parse that.
auto val = obj.get_value_of_key({key.data(), key.length()});
if (val.get_type() != ::sajson::TYPE_STRING) { throw "field not a string"; }
@@ -40,7 +38,7 @@ struct sajson {
if (endptr != &str[val.get_string_length()]) { throw "field is a string, but not an integer string"; }
return result;
}
simdjson_inline uint64_t get_nullable_str_uint64(const ::sajson::value &obj, std::string_view key) {
simdjson_really_inline uint64_t get_nullable_str_uint64(const ::sajson::value &obj, std::string_view key) {
auto val = obj.get_value_of_key({key.data(), key.length()});
if (val.get_type() == ::sajson::TYPE_NULL) { return 0; }
if (val.get_type() != ::sajson::TYPE_STRING) { throw "field not a string"; }
@@ -50,7 +48,7 @@ struct sajson {
if (endptr != &str[val.get_string_length()]) { throw "field is a string, but not an integer string"; }
return result;
}
simdjson_inline partial_tweets::twitter_user<std::string_view> get_user(const ::sajson::value &obj, std::string_view key) {
simdjson_really_inline partial_tweets::twitter_user<std::string_view> get_user(const ::sajson::value &obj, std::string_view key) {
auto user = obj.get_value_of_key({key.data(), key.length()});
if (user.get_type() != ::sajson::TYPE_OBJECT) { throw "user is not an object"; }
return { get_str_uint64(user, "id_str"), get_string_view(user, "screen_name") };
+2 -2
View File
@@ -13,7 +13,7 @@ struct simdjson_dom {
dom::parser parser{};
simdjson_inline uint64_t nullable_int(dom::element element) {
simdjson_really_inline uint64_t nullable_int(dom::element element) {
if (element.is_null()) { return 0; }
return element;
}
@@ -40,4 +40,4 @@ BENCHMARK_TEMPLATE(partial_tweets, simdjson_dom)->UseManualTime();
} // namespace partial_tweets
#endif // SIMDJSON_EXCEPTIONS
#endif // SIMDJSON_EXCEPTIONS
+2 -2
View File
@@ -13,12 +13,12 @@ struct simdjson_ondemand {
ondemand::parser parser{};
simdjson_inline uint64_t nullable_int(ondemand::value value) {
simdjson_really_inline uint64_t nullable_int(ondemand::value value) {
if (value.is_null()) { return 0; }
return value;
}
simdjson_inline twitter_user<std::string_view> read_user(ondemand::object user) {
simdjson_really_inline twitter_user<std::string_view> read_user(ondemand::object user) {
return { user.find_field("id"), user.find_field("screen_name") };
}
+2 -2
View File
@@ -32,7 +32,7 @@ struct tweet {
uint64_t retweet_count{};
uint64_t favorite_count{};
template<typename OtherStringType>
simdjson_inline bool operator==(const tweet<OtherStringType> &other) const {
simdjson_really_inline bool operator==(const tweet<OtherStringType> &other) const {
return created_at == other.created_at &&
id == other.id &&
result == other.result &&
@@ -42,7 +42,7 @@ struct tweet {
favorite_count == other.favorite_count;
}
template<typename OtherStringType>
simdjson_inline bool operator!=(const tweet<OtherStringType> &other) const { return !(*this == other); }
simdjson_really_inline bool operator!=(const tweet<OtherStringType> &other) const { return !(*this == other); }
};
template<typename StringType>
+6 -6
View File
@@ -9,24 +9,24 @@ namespace partial_tweets {
struct yyjson_base {
using StringType=std::string_view;
simdjson_inline std::string_view get_string_view(yyjson_val *obj, std::string_view key) {
simdjson_really_inline std::string_view get_string_view(yyjson_val *obj, std::string_view key) {
auto val = yyjson_obj_getn(obj, key.data(), key.length());
if (!yyjson_is_str(val)) { throw "field is not uint64 or null!"; }
return { yyjson_get_str(val), yyjson_get_len(val) };
}
simdjson_inline uint64_t get_uint64(yyjson_val *obj, std::string_view key) {
simdjson_really_inline uint64_t get_uint64(yyjson_val *obj, std::string_view key) {
auto val = yyjson_obj_getn(obj, key.data(), key.length());
if (!yyjson_is_uint(val)) { throw "field is not uint64 or null!"; }
return yyjson_get_uint(val);
}
simdjson_inline uint64_t get_nullable_uint64(yyjson_val *obj, std::string_view key) {
simdjson_really_inline uint64_t get_nullable_uint64(yyjson_val *obj, std::string_view key) {
auto val = yyjson_obj_getn(obj, key.data(), key.length());
if (!yyjson_is_uint(val)) { }
auto type = yyjson_get_type(val);
if (type != YYJSON_TYPE_NUM && type != YYJSON_TYPE_NULL ) { throw "field is not uint64 or null!"; }
return yyjson_get_uint(val);
}
simdjson_inline partial_tweets::twitter_user<std::string_view> get_user(yyjson_val *obj, std::string_view key) {
simdjson_really_inline partial_tweets::twitter_user<std::string_view> get_user(yyjson_val *obj, std::string_view key) {
auto user = yyjson_obj_getn(obj, key.data(), key.length());
if (!yyjson_is_obj(user)) { throw "missing twitter user field!"; }
return { get_uint64(user, "id"), get_string_view(user, "screen_name") };
@@ -66,14 +66,14 @@ struct yyjson : yyjson_base {
}
};
BENCHMARK_TEMPLATE(partial_tweets, yyjson)->UseManualTime();
#if SIMDJSON_COMPETITION_ONDEMAND_INSITU
struct yyjson_insitu : yyjson_base {
bool run(simdjson::padded_string &json, std::vector<tweet<std::string_view>> &result) {
return yyjson_base::run(yyjson_read_opts(json.data(), json.size(), YYJSON_READ_INSITU, 0, 0), result);
}
};
BENCHMARK_TEMPLATE(partial_tweets, yyjson_insitu)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_INSITU
} // namespace partial_tweets
#endif // SIMDJSON_COMPETITION_YYJSON
+1 -1
View File
@@ -48,7 +48,7 @@ struct nlohmann_json_sax {
}
bool number_unsigned(number_unsigned_t val) override {
if (values & key_rt && !(values & found_rt)) { // retweet_count
rt = int(val);
rt = val;
values &= ~(key_rt);
values |= (found_rt);
if (rt <= max_rt && rt >= result.retweet_count) { // Check if current tweet has more retweet than previous top tweet
+3 -3
View File
@@ -56,14 +56,14 @@ struct rapidjson : rapidjson_base {
}
};
BENCHMARK_TEMPLATE(top_tweet, rapidjson)->UseManualTime();
#if SIMDJSON_COMPETITION_ONDEMAND_INSITU
struct rapidjson_insitu : rapidjson_base {
bool run(simdjson::padded_string &json, int64_t max_retweet_count, top_tweet_result<StringType> &result) {
return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag|kParseInsituFlag>(json.data()), max_retweet_count, result);
return rapidjson_base::run(doc.ParseInsitu<kParseValidateEncodingFlag>(json.data()), max_retweet_count, result);
}
};
BENCHMARK_TEMPLATE(top_tweet, rapidjson_insitu)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_INSITU
} // namespace top_tweet
#endif // SIMDJSON_COMPETITION_RAPIDJSON
-1
View File
@@ -11,7 +11,6 @@ struct sajson {
size_t ast_buffer_size{0};
size_t *ast_buffer{nullptr};
~sajson() { free(ast_buffer); }
bool run(simdjson::padded_string &json, int32_t max_retweet_count, top_tweet_result<StringType> &result) {
if (!ast_buffer) {
+3 -3
View File
@@ -13,13 +13,13 @@ struct top_tweet_result {
StringType screen_name{};
StringType text{};
template<typename OtherStringType>
simdjson_inline bool operator==(const top_tweet_result<OtherStringType> &other) const {
simdjson_really_inline bool operator==(const top_tweet_result<OtherStringType> &other) const {
return retweet_count == other.retweet_count &&
screen_name == other.screen_name &&
text == other.text;
}
template<typename OtherStringType>
simdjson_inline bool operator!=(const top_tweet_result<OtherStringType> &other) const { return !(*this == other); }
simdjson_really_inline bool operator!=(const top_tweet_result<OtherStringType> &other) const { return !(*this == other); }
};
template<typename StringType>
@@ -60,7 +60,7 @@ struct runner : public file_runner<I> {
struct simdjson_dom;
template<typename I> simdjson_inline static void top_tweet(benchmark::State &state) {
template<typename I> simdjson_really_inline static void top_tweet(benchmark::State &state) {
json_benchmark::run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
}
+2 -2
View File
@@ -55,14 +55,14 @@ struct yyjson : yyjson_base {
}
};
BENCHMARK_TEMPLATE(top_tweet, yyjson)->UseManualTime();
#if SIMDJSON_COMPETITION_ONDEMAND_INSITU
struct yyjson_insitu : yyjson_base {
bool run(simdjson::padded_string &json, int64_t max_retweet_count, top_tweet_result<StringType> &result) {
return yyjson_base::run(yyjson_read_opts(json.data(), json.size(), YYJSON_READ_INSITU, 0, 0), max_retweet_count, result);
}
};
BENCHMARK_TEMPLATE(top_tweet, yyjson_insitu)->UseManualTime();
#endif // SIMDJSON_COMPETITION_ONDEMAND_INSITU
} // namespace top_tweet
#endif // SIMDJSON_COMPETITION_YYJSON
-23
View File
@@ -1,23 +0,0 @@
# This module provides function for joining paths
# known from most languages
#
# SPDX-License-Identifier: (MIT OR CC0-1.0)
# Copyright 2020 Jan Tojnar
# https://github.com/jtojnar/cmake-snips
#
# Modelled after Pythons os.path.join
# https://docs.python.org/3.7/library/os.path.html#os.path.join
# Windows not supported
function(join_paths joined_path first_path_segment)
set(temp_path "${first_path_segment}")
foreach(current_segment IN LISTS ARGN)
if(NOT ("${current_segment}" STREQUAL ""))
if(IS_ABSOLUTE "${current_segment}")
set(temp_path "${current_segment}")
else()
set(temp_path "${temp_path}/${current_segment}")
endif()
endif()
endforeach()
set(${joined_path} "${temp_path}" PARENT_SCOPE)
endfunction()

Some files were not shown because too many files have changed in this diff Show More