mirror of
https://github.com/simdjson/simdjson
synced 2026-06-08 17:27:07 +00:00
Compare commits
54 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7edd32dc6b | |||
| 91b74c593c | |||
| fa629bd783 | |||
| 7784aa86c2 | |||
| a47b64e0ff | |||
| d92bc82b37 | |||
| 5b02e117bb | |||
| ef6fc77861 | |||
| e3cac71afe | |||
| cbe562c1d5 | |||
| 30437d0ff6 | |||
| 1e97af7dfb | |||
| a74e87df17 | |||
| 3274299647 | |||
| 22ca25faa6 | |||
| 37399fd41b | |||
| a170145d5b | |||
| 7d1d635883 | |||
| ddfa8ae928 | |||
| 25dfb078c7 | |||
| 9f0710e3ad | |||
| 19b9cd9b05 | |||
| 7540248396 | |||
| ab09e96de9 | |||
| 66252e4925 | |||
| 944778c186 | |||
| 62868a2f53 | |||
| 3c446c551b | |||
| efe3d4b7d7 | |||
| 28db262b47 | |||
| bf076bfb5f | |||
| 7bf1ecff9e | |||
| e25a40facf | |||
| a442842f0d | |||
| 421ed4f390 | |||
| 7dff5fa37c | |||
| 9a0527c380 | |||
| b160cf0610 | |||
| 553ee69b7c | |||
| bf70cd0ebf | |||
| cf0551edf5 | |||
| ddb094d550 | |||
| 975fdda03c | |||
| c5c43e9c7f | |||
| 031ee32969 | |||
| b613326d69 | |||
| 53dad74d8e | |||
| e4da19f5cf | |||
| 342b58986a | |||
| 7edc475886 | |||
| d2f8e84fc1 | |||
| 08000518cb | |||
| 90040d96a5 | |||
| 2934892c0b |
@@ -0,0 +1,25 @@
|
||||
CompileFlags:
|
||||
CompilationDatabase: build
|
||||
Diagnostics:
|
||||
Suppress:
|
||||
- misc-unused-alias-decls
|
||||
- misc-unused-using-decls
|
||||
- misc-definitions-in-headers # TODO fix and remove these violations
|
||||
---
|
||||
If:
|
||||
PathMatch:
|
||||
- include/.*
|
||||
- src/.*
|
||||
PathExclude:
|
||||
- include/simdjson.h
|
||||
- src/simdjson.cpp
|
||||
CompileFlags:
|
||||
Add:
|
||||
- -Wno-unneeded-internal-declaration
|
||||
- -Wno-undefined-inline # TODO fix and remove these violations
|
||||
- -Wno-undefined-internal # TODO fix and remove these violations
|
||||
- -Wno-unused-function
|
||||
- -Wno-unused-const-variable
|
||||
Diagnostics:
|
||||
Suppress:
|
||||
- pp_including_mainfile_in_preamble
|
||||
@@ -31,4 +31,4 @@ jobs:
|
||||
./alpine.sh cmake --build build_for_alpine
|
||||
- name: test
|
||||
run: |
|
||||
./alpine.sh bash -c "cd build_for_alpine && ctest -LE explicitonly"
|
||||
./alpine.sh bash -c "cd build_for_alpine && ctest -LE explicitonly --output-on-failure"
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
# fuzzers that change behaviour with SIMDJSON_FORCE_IMPLEMENTATION
|
||||
defaultimplfuzzers: atpointer dump dump_raw_tape element minify parser print_json
|
||||
# fuzzers that loop over the implementations themselves, or don't need to switch.
|
||||
implfuzzers: implementations minifyimpl ndjson ondemand padded utf8
|
||||
implfuzzers: implementations minifyimpl ndjson ondemand padded utf8 # TODO add singlestage
|
||||
implementations: haswell westmere fallback
|
||||
UBSAN_OPTIONS: halt_on_error=1
|
||||
MAXLEN: -max_len=4000
|
||||
@@ -124,7 +124,7 @@ jobs:
|
||||
done
|
||||
|
||||
- name: Save the corpus as a github artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: corpus
|
||||
path: corpus.tar
|
||||
@@ -147,7 +147,7 @@ jobs:
|
||||
run: tar cf valgrind.tar valgrind-*.txt
|
||||
|
||||
- name: Save valgrind output as a github artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
if: always()
|
||||
with:
|
||||
name: valgrindresults
|
||||
@@ -155,7 +155,7 @@ jobs:
|
||||
if-no-files-found: ignore
|
||||
|
||||
- name: Archive any crashes as an artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
if: always()
|
||||
with:
|
||||
name: crashes
|
||||
|
||||
@@ -20,13 +20,13 @@ jobs:
|
||||
cd builddebug &&
|
||||
cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly &&
|
||||
ctest --output-on-failure -LE explicitonly -j &&
|
||||
cd .. &&
|
||||
mkdir build &&
|
||||
cd build &&
|
||||
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly &&
|
||||
ctest --output-on-failure -LE explicitonly -j &&
|
||||
cmake --install . &&
|
||||
echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
|
||||
cd ../tests/installation_tests/find &&
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
runs-on: ubuntu-20.04
|
||||
name: Build on ubuntu-20.04 ppc64le
|
||||
steps:
|
||||
- uses: actions/checkout@v2.1.0
|
||||
- uses: actions/checkout@v3
|
||||
- uses: uraimo/run-on-arch-action@v2.0.5
|
||||
name: Run commands
|
||||
id: runcmd
|
||||
|
||||
@@ -25,13 +25,13 @@ jobs:
|
||||
cd builddebug &&
|
||||
cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly &&
|
||||
ctest --output-on-failure -LE explicitonly -j &&
|
||||
cd .. &&
|
||||
mkdir build &&
|
||||
cd build &&
|
||||
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly &&
|
||||
ctest --output-on-failure -LE explicitonly -j &&
|
||||
cmake --install . &&
|
||||
echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
|
||||
cd ../tests/installation_tests/find &&
|
||||
|
||||
@@ -20,13 +20,13 @@ jobs:
|
||||
cd builddebug &&
|
||||
cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_EXCEPTIONS=OFF -DBUILD_SHARED_LIBS=OFF .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly &&
|
||||
ctest --output-on-failure -LE explicitonly -j &&
|
||||
cd .. &&
|
||||
mkdir build &&
|
||||
cd build &&
|
||||
cmake -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_EXCEPTIONS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly &&
|
||||
ctest --output-on-failure -LE explicitonly -j &&
|
||||
make install &&
|
||||
echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
|
||||
mkdir testfindpackage &&
|
||||
|
||||
@@ -20,13 +20,13 @@ jobs:
|
||||
cd builddebug &&
|
||||
cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly &&
|
||||
ctest --output-on-failure -LE explicitonly -j &&
|
||||
cd .. &&
|
||||
mkdir build &&
|
||||
cd build &&
|
||||
cmake -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_ENABLE_THREADS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly &&
|
||||
ctest --output-on-failure -LE explicitonly -j &&
|
||||
make install &&
|
||||
echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
|
||||
mkdir testfindpackage &&
|
||||
|
||||
@@ -20,11 +20,11 @@ jobs:
|
||||
cd builddebug &&
|
||||
cmake -DSIMDJSON_SANITIZE=ON -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly
|
||||
ctest --output-on-failure -LE explicitonly -j
|
||||
- name: Use cmake with undefined sanitizer
|
||||
run: |
|
||||
mkdir builddebugundefsani &&
|
||||
cd builddebugundefsani &&
|
||||
cmake -DSIMDJSON_SANITIZE_UNDEFINED=ON -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly
|
||||
ctest --output-on-failure -LE explicitonly -j
|
||||
|
||||
@@ -20,13 +20,13 @@ jobs:
|
||||
cd builddebug &&
|
||||
cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly &&
|
||||
ctest --output-on-failure -LE explicitonly -j &&
|
||||
cd .. &&
|
||||
mkdir build &&
|
||||
cd build &&
|
||||
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly &&
|
||||
ctest --output-on-failure -LE explicitonly -j &&
|
||||
cmake --install . &&
|
||||
echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
|
||||
cd ../tests/installation_tests/find &&
|
||||
|
||||
@@ -22,4 +22,4 @@ jobs:
|
||||
cd build &&
|
||||
CXX=clang++-13 cmake -DSIMDJSON_DEVELOPER_MODE=ON .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly
|
||||
ctest --output-on-failure -LE explicitonly -j
|
||||
@@ -22,4 +22,4 @@ jobs:
|
||||
cd build &&
|
||||
CXX=clang++-14 cmake -DSIMDJSON_DEVELOPER_MODE=ON .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly
|
||||
ctest --output-on-failure -LE explicitonly -j
|
||||
|
||||
@@ -9,8 +9,8 @@ jobs:
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip github]')
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/cache@v2
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
path: dependencies/.cache
|
||||
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
|
||||
@@ -20,13 +20,13 @@ jobs:
|
||||
cd builddebug &&
|
||||
CXX=g++-12 cmake -DSIMDJSON_CXX_STANDARD=20 -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly &&
|
||||
ctest --output-on-failure -LE explicitonly -j &&
|
||||
cd .. &&
|
||||
mkdir build &&
|
||||
cd build &&
|
||||
CXX=g++-12 cmake -DSIMDJSON_CXX_STANDARD=20 -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly &&
|
||||
ctest --output-on-failure -LE explicitonly -j &&
|
||||
cmake --install . &&
|
||||
echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
|
||||
cd ../tests/installation_tests/find &&
|
||||
|
||||
@@ -22,4 +22,4 @@ jobs:
|
||||
cd build &&
|
||||
CXX=g++-12 cmake -DSIMDJSON_DEVELOPER_MODE=ON .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly
|
||||
ctest --output-on-failure -LE explicitonly -j
|
||||
@@ -20,13 +20,13 @@ jobs:
|
||||
cd builddebug &&
|
||||
cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly &&
|
||||
ctest --output-on-failure -LE explicitonly -j &&
|
||||
cd .. &&
|
||||
mkdir build &&
|
||||
cd build &&
|
||||
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly &&
|
||||
ctest --output-on-failure -LE explicitonly -j &&
|
||||
cmake --install . &&
|
||||
echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
|
||||
cd ../tests/installation_tests/find &&
|
||||
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
- {gen: Visual Studio 17 2022, arch: x64, shared: OFF}
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
- name: Configure
|
||||
run: |
|
||||
cmake -DSIMDJSON_CXX_STANDARD=20 -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build
|
||||
|
||||
@@ -104,3 +104,6 @@ objs
|
||||
!.vscode/tasks.json
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
|
||||
# clangd
|
||||
.cache
|
||||
Vendored
+8
-3
@@ -4,10 +4,15 @@
|
||||
|
||||
// List of extensions which should be recommended for users of this workspace.
|
||||
"recommendations": [
|
||||
// Syntax
|
||||
"ms-vscode.cpptools",
|
||||
"ms-vscode.cmake-tools",
|
||||
// C++
|
||||
"llvm-vs-code-extensions.vscode-clangd",
|
||||
"xaver.clang-format",
|
||||
// Python
|
||||
"ms-python.python",
|
||||
// .github/*
|
||||
"github.vscode-github-actions",
|
||||
// cmake
|
||||
"ms-vscode.cmake-tools",
|
||||
"twxs.cmake"
|
||||
],
|
||||
// List of extensions recommended by VS Code that should not be recommended for users of this workspace.
|
||||
|
||||
Vendored
+1
@@ -5,6 +5,7 @@
|
||||
],
|
||||
"files.trimTrailingWhitespace": true,
|
||||
"files.associations": {
|
||||
".clangd": "yaml",
|
||||
"array": "cpp",
|
||||
"iterator": "cpp",
|
||||
"chrono": "cpp",
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
|
||||
project(
|
||||
simdjson
|
||||
# The version number is modified by tools/release.py
|
||||
VERSION 3.2.0
|
||||
VERSION 3.2.1
|
||||
DESCRIPTION "Parsing gigabytes of JSON per second"
|
||||
HOMEPAGE_URL "https://simdjson.org/"
|
||||
LANGUAGES CXX C
|
||||
|
||||
@@ -38,7 +38,7 @@ PROJECT_NAME = simdjson
|
||||
# could be handy for archiving the generated documentation or if some version
|
||||
# control system is used.
|
||||
|
||||
PROJECT_NUMBER = "3.2.0"
|
||||
PROJECT_NUMBER = "3.2.1"
|
||||
|
||||
# Using the PROJECT_BRIEF tag one can provide an optional one line description
|
||||
# for a project that appears at the top of each page and should give viewer a
|
||||
|
||||
+35
-17
@@ -6,16 +6,26 @@ Here is wisdom about how to build, test and run simdjson from within the reposit
|
||||
|
||||
If you plan to contribute to simdjson, please read our [CONTRIBUTING](https://github.com/simdjson/simdjson/blob/master/CONTRIBUTING.md) guide.
|
||||
|
||||
|
||||
- [Hacking simdjson](#hacking-simdjson)
|
||||
- [Build Quickstart](#build-quickstart)
|
||||
- [Design notes](#design-notes)
|
||||
- [Developer mode](#developer-mode)
|
||||
- [Directory Structure and Source](#directory-structure-and-source)
|
||||
- [Runtime Dispatching](#runtime-dispatching)
|
||||
- [Regenerating Single-Header Files](#regenerating-single-header-files)
|
||||
- [Usage (CMake on 64-bit platforms like Linux, FreeBSD or macOS)](#usage-cmake-on-64-bit-platforms-like-linux-freebsd-or-macos)
|
||||
- [Usage (CMake on 64-bit Windows using Visual Studio 2019)](#usage-cmake-on-64-bit-windows-using-visual-studio-2019)
|
||||
- [Usage (CMake on 64-bit Windows using Visual Studio 2019 or better)](#usage-cmake-on-64-bit-windows-using-visual-studio-2019-or-better)
|
||||
- [Various References](#various-references)
|
||||
|
||||
Build Quickstart
|
||||
------------------------------
|
||||
|
||||
```bash
|
||||
mkdir build
|
||||
cd build
|
||||
cmake -D SIMDJSON_DEVELOPER_MODE=ON ..
|
||||
cmake --build .
|
||||
```
|
||||
|
||||
Design notes
|
||||
------------------------------
|
||||
@@ -63,26 +73,34 @@ simdjson's source structure, from the top level, looks like this:
|
||||
|
||||
* **CMakeLists.txt:** The main build system.
|
||||
* **include:** User-facing declarations and inline definitions (most user-facing functions are inlined).
|
||||
* simdjson.h: A "main include" that includes files from include/simdjson/. This is equivalent to
|
||||
* simdjson.h: the `simdjson` namespace. A "main include" that includes files from include/simdjson/. This is equivalent to
|
||||
the distributed simdjson.h.
|
||||
* simdjson/*.h: Declarations for public simdjson classes and functions.
|
||||
* simdjson/*-inl.h: Definitions for public simdjson classes and functions.
|
||||
* simdjson/*.h: Declarations for public simdjson classes and functions.
|
||||
* simdjson/*-inl.h: Definitions for public simdjson classes and functions.
|
||||
* simdjson/internal/*.h: the `simdjson::internal` namespace. Private classes and functions used by the rest of simdjson.
|
||||
* simdjson/dom.h: the `simdjson::dom` namespace. Includes all public DOM classes.
|
||||
* simdjson/dom/*.h: Declarations/definitions for individual DOM classes.
|
||||
* simdjson/arm64|fallback|haswell|icelake|ppc64|westmere.h: `simdjson::<implementation>` namesapce. Common implementation-specific tools like number and string parsing, as well as minification.
|
||||
* simdjson/arm64|fallback|haswell|icelake|ppc64|westmere/*.h: implementation-specific functions such as , etc.
|
||||
* simdjson/generic/*.h: the bulk of the actual code, written generically and compiled for each implementation, using functions defined in the implementation's .h files.
|
||||
* simdjson/generic/dependencies.h: dependencies on common, non-implementation-specific simdjson classes. This will be included before including amalgamated.h.
|
||||
* simdjson/generic/amalgamated.h: all generic ondemand classes for an implementation.
|
||||
* simdjson/ondemand.h: the `simdjson::ondemand` namespace. Includes all public ondemand classes.
|
||||
* simdjson/builtin.h: the `simdjson::builtin` namespace. Aliased to the most universal implementation available.
|
||||
* simdjson/builtin/ondemand.h: the `simdjson::builtin::ondemand` namespace.
|
||||
* simdjson/arm64|fallback|haswell|icelake|ppc64|westmere/ondemand.h: the `simdjson::<implementation>::ondemand` namespace. on demand compiled for the specific implementation.
|
||||
* simdjson/generic/ondemand/*.h: individual on demand classes, generically written.
|
||||
* simdjson/generic/ondemand/dependencies.h: dependencies on common, non-implementation-specific simdjson classes. This will be included before including amalgamated.h.
|
||||
* simdjson/generic/ondemand/amalgamated.h: all generic ondemand classes for an implementation.
|
||||
* **src:** The source files for non-inlined functionality (e.g. the architecture-specific parser
|
||||
implementations).
|
||||
* simdjson.cpp: A "main source" that includes all implementation files from src/. This is
|
||||
equivalent to the distributed simdjson.cpp.
|
||||
* arm64/|fallback/|haswell/|ppc64/|westmere/: Architecture-specific implementations. All functions are
|
||||
Each architecture defines its own namespace, e.g. simdjson::haswell.
|
||||
* generic/: Generic implementations of the simdjson parser. These files may be included and
|
||||
compiled multiple times, from whichever architectures use them. They assume they are already
|
||||
enclosed in a namespace, e.g.:
|
||||
```c++
|
||||
namespace simdjson {
|
||||
namespace haswell {
|
||||
#include "generic/stage1/json_structural_indexer.h"
|
||||
}
|
||||
}
|
||||
```
|
||||
* *.cpp: other misc. implementations, such as `simdjson::implementation` and the minifier.
|
||||
* arm64|fallback|haswell|icelake|ppc64|westmere.cpp: Architecture-specific parser implementations.
|
||||
* generic/*.h: `simdjson::<implementation>` namespace. Generic implementation of the parser, particularly the `dom_parser_implementation`.
|
||||
* generic/stage1/*.h: `simdjson::<implementation>::stage1` namespace. Generic implementation of the simd-heavy tokenizer/indexer pass of the simdjson parser. Used for the On Demand interface
|
||||
* generic/stage2/*.h: `simdjson::<implementation>::stage2` namespace. Generic implementation of the tape creator, which consumes the index from stage 1 and actually parses numbers and string and such. Used for the DOM interface.
|
||||
|
||||
Other important files and directories:
|
||||
* **.drone.yml:** Definitions for Drone CI.
|
||||
|
||||
@@ -57,6 +57,7 @@ Real-world usage
|
||||
- [fastgltf](https://github.com/spnda/fastgltf)
|
||||
- [vast](https://github.com/tenzir/vast)
|
||||
- [ada-url](https://github.com/ada-url/ada)
|
||||
- [fastgron](https://github.com/adamritter/fastgron)
|
||||
|
||||
If you are planning to use simdjson in a product, please work from one of our releases.
|
||||
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
#pragma once
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
|
||||
#include "amazon_cellphones.h"
|
||||
|
||||
namespace amazon_cellphones {
|
||||
|
||||
using namespace simdjson;
|
||||
|
||||
template<bool threaded>
|
||||
struct simdjson_singlestage {
|
||||
using StringType = std::string;
|
||||
|
||||
singlestage::parser parser{};
|
||||
|
||||
bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
|
||||
#ifdef SIMDJSON_THREADS_ENABLED
|
||||
parser.threaded = threaded;
|
||||
#endif
|
||||
singlestage::document_stream stream = parser.iterate_many(json);
|
||||
singlestage::document_stream::iterator i = stream.begin();
|
||||
++i; // Skip first line
|
||||
for (;i != stream.end(); ++i) {
|
||||
auto doc = *i;
|
||||
size_t index{0};
|
||||
StringType copy;
|
||||
double rating;
|
||||
uint64_t reviews;
|
||||
for ( auto value : doc ) {
|
||||
switch (index)
|
||||
{
|
||||
case 1:
|
||||
copy = StringType(std::string_view(value));
|
||||
break;
|
||||
case 5:
|
||||
rating = double(value);
|
||||
break;
|
||||
case 7:
|
||||
reviews = uint64_t(value);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
index++;
|
||||
}
|
||||
|
||||
auto x = result.find(copy);
|
||||
if (x == result.end()) { // If key not found, add new key
|
||||
result.emplace(copy, amazon_cellphones::brand{
|
||||
rating * reviews,
|
||||
reviews
|
||||
});
|
||||
} else { // Otherwise, update key data
|
||||
x->second.cumulative_rating += rating * reviews;
|
||||
x->second.reviews_count += reviews;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_singlestage<UNTHREADED>)->UseManualTime();
|
||||
#ifdef SIMDJSON_THREADS_ENABLED
|
||||
BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_singlestage<THREADED>)->UseManualTime();
|
||||
#endif
|
||||
|
||||
} // namespace amazon_cellphones
|
||||
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
@@ -26,6 +26,7 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
|
||||
|
||||
SIMDJSON_POP_DISABLE_WARNINGS
|
||||
#include "json2msgpack/simdjson_ondemand.h"
|
||||
#include "json2msgpack/simdjson_singlestage.h"
|
||||
#include "json2msgpack/simdjson_dom.h"
|
||||
#include "json2msgpack/yyjson.h"
|
||||
#include "json2msgpack/rapidjson.h"
|
||||
@@ -35,6 +36,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
|
||||
#include "json2msgpack/nlohmann_json.h"
|
||||
|
||||
#include "partial_tweets/simdjson_ondemand.h"
|
||||
#include "partial_tweets/simdjson_singlestage.h"
|
||||
#include "partial_tweets/simdjson_dom.h"
|
||||
#include "partial_tweets/yyjson.h"
|
||||
#if SIMDJSON_COMPETITION_ONDEMAND_SAJSON
|
||||
@@ -52,6 +54,8 @@ SIMDJSON_POP_DISABLE_WARNINGS
|
||||
|
||||
#include "distinct_user_id/simdjson_ondemand.h"
|
||||
#include "distinct_user_id/simdjson_ondemand_json_pointer.h"
|
||||
#include "distinct_user_id/simdjson_singlestage.h"
|
||||
#include "distinct_user_id/simdjson_singlestage_json_pointer.h"
|
||||
#include "distinct_user_id/simdjson_dom.h"
|
||||
#include "distinct_user_id/simdjson_dom_json_pointer.h"
|
||||
#include "distinct_user_id/yyjson.h"
|
||||
@@ -68,6 +72,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
|
||||
#endif // SIMDJSON_COMPETITION_SAX
|
||||
|
||||
#include "find_tweet/simdjson_ondemand.h"
|
||||
#include "find_tweet/simdjson_singlestage.h"
|
||||
#include "find_tweet/simdjson_dom.h"
|
||||
#include "find_tweet/yyjson.h"
|
||||
#if SIMDJSON_COMPETITION_ONDEMAND_SAJSON
|
||||
@@ -83,6 +88,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
|
||||
#endif // SIMDJSON_COMPETITION_SAX
|
||||
|
||||
#include "top_tweet/simdjson_ondemand.h"
|
||||
#include "top_tweet/simdjson_singlestage.h"
|
||||
#include "top_tweet/simdjson_dom.h"
|
||||
#include "top_tweet/yyjson.h"
|
||||
#if SIMDJSON_COMPETITION_ONDEMAND_SAJSON
|
||||
@@ -99,6 +105,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
|
||||
|
||||
|
||||
#include "kostya/simdjson_ondemand.h"
|
||||
#include "kostya/simdjson_singlestage.h"
|
||||
#include "kostya/simdjson_dom.h"
|
||||
#include "kostya/yyjson.h"
|
||||
#if SIMDJSON_COMPETITION_ONDEMAND_SAJSON
|
||||
@@ -117,6 +124,10 @@ SIMDJSON_POP_DISABLE_WARNINGS
|
||||
#if SIMDJSON_COMPETITION_ONDEMAND_UNORDERED
|
||||
#include "large_random/simdjson_ondemand_unordered.h"
|
||||
#endif // SIMDJSON_COMPETITION_ONDEMAND_UNORDERED
|
||||
#include "large_random/simdjson_singlestage.h"
|
||||
#if SIMDJSON_COMPETITION_SINGLESTAGE_UNORDERED
|
||||
#include "large_random/simdjson_singlestage_unordered.h"
|
||||
#endif // SIMDJSON_COMPETITION_SINGLESTAGE_UNORDERED
|
||||
#include "large_random/simdjson_dom.h"
|
||||
#include "large_random/yyjson.h"
|
||||
#if SIMDJSON_COMPETITION_ONDEMAND_SAJSON
|
||||
@@ -133,8 +144,10 @@ SIMDJSON_POP_DISABLE_WARNINGS
|
||||
|
||||
#include "amazon_cellphones/simdjson_dom.h"
|
||||
#include "amazon_cellphones/simdjson_ondemand.h"
|
||||
#include "amazon_cellphones/simdjson_singlestage.h"
|
||||
|
||||
#include "large_amazon_cellphones/simdjson_dom.h"
|
||||
#include "large_amazon_cellphones/simdjson_ondemand.h"
|
||||
#include "large_amazon_cellphones/simdjson_singlestage.h"
|
||||
|
||||
BENCHMARK_MAIN();
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
#pragma once
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
|
||||
#include "distinct_user_id.h"
|
||||
|
||||
namespace distinct_user_id {
|
||||
|
||||
using namespace simdjson;
|
||||
|
||||
struct simdjson_singlestage {
|
||||
singlestage::parser parser{};
|
||||
|
||||
bool run(simdjson::padded_string &json, std::vector<uint64_t> &result) {
|
||||
// Walk the document, parsing as we go
|
||||
auto doc = parser.iterate(json);
|
||||
for (singlestage::object tweet : doc.find_field("statuses")) {
|
||||
// We believe that all statuses have a matching
|
||||
// user, and we are willing to throw when they do not.
|
||||
result.push_back(tweet.find_field("user").find_field("id"));
|
||||
// Not all tweets have a "retweeted_status", but when they do
|
||||
// we want to go and find the user within.
|
||||
auto retweet = tweet.find_field("retweeted_status");
|
||||
if (!retweet.error()) {
|
||||
result.push_back(retweet.find_field("user").find_field("id"));
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
BENCHMARK_TEMPLATE(distinct_user_id, simdjson_singlestage)->UseManualTime();
|
||||
|
||||
} // namespace distinct_user_id
|
||||
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
@@ -0,0 +1,37 @@
|
||||
#pragma once
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
|
||||
#include "distinct_user_id.h"
|
||||
|
||||
namespace distinct_user_id {
|
||||
|
||||
using namespace simdjson;
|
||||
|
||||
struct simdjson_singlestage_json_pointer {
|
||||
singlestage::parser parser{};
|
||||
|
||||
bool run(simdjson::padded_string &json, std::vector<uint64_t> &result) {
|
||||
// Walk the document, parsing as we go
|
||||
auto doc = parser.iterate(json);
|
||||
for (singlestage::object tweet : doc.find_field("statuses")) {
|
||||
// We believe that all statuses have a matching
|
||||
// user, and we are willing to throw when they do not.
|
||||
result.push_back(tweet.at_pointer("/user/id"));
|
||||
// Not all tweets have a "retweeted_status", but when they do
|
||||
// we want to go and find the user within.
|
||||
auto retweet_id = tweet.at_pointer("/retweeted_status/user/id");
|
||||
if (retweet_id.error() != NO_SUCH_FIELD) {
|
||||
result.push_back(retweet_id);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
BENCHMARK_TEMPLATE(distinct_user_id, simdjson_singlestage_json_pointer)->UseManualTime();
|
||||
|
||||
} // namespace distinct_user_id
|
||||
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
@@ -0,0 +1,33 @@
|
||||
#pragma once
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
|
||||
#include "find_tweet.h"
|
||||
|
||||
namespace find_tweet {
|
||||
|
||||
using namespace simdjson;
|
||||
|
||||
struct simdjson_singlestage {
|
||||
using StringType=std::string_view;
|
||||
|
||||
singlestage::parser parser{};
|
||||
|
||||
bool run(simdjson::padded_string &json, uint64_t find_id, std::string_view &result) {
|
||||
// Walk the document, parsing as we go
|
||||
auto doc = parser.iterate(json);
|
||||
for (auto tweet : doc.find_field("statuses")) {
|
||||
if (uint64_t(tweet.find_field("id")) == find_id) {
|
||||
result = tweet.find_field("text");
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
BENCHMARK_TEMPLATE(find_tweet, simdjson_singlestage)->UseManualTime();
|
||||
|
||||
} // namespace find_tweet
|
||||
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
@@ -15,7 +15,7 @@ struct simdjsondom2msgpack {
|
||||
* @param json JSON input
|
||||
* @param buf temporary buffer (must be large enough, with simdjson::SIMDJSON_PADDING bytes
|
||||
* of padding)
|
||||
* @return std::string_view msgpack output, writting to the temporary buffer
|
||||
* @return std::string_view msgpack output, writing to the temporary buffer
|
||||
*/
|
||||
inline std::string_view to_msgpack(const simdjson::padded_string &json,
|
||||
uint8_t *buf);
|
||||
|
||||
@@ -8,7 +8,7 @@ namespace json2msgpack {
|
||||
using namespace simdjson;
|
||||
|
||||
/**
|
||||
* @brief The simdjson2msgpack struct is used to quickly convert
|
||||
* @brief The simdjsonondemand2msgpack struct is used to quickly convert
|
||||
* JSON strings to msgpack views. You must provide a pointer to
|
||||
* a large memory region where the msgpack gets written. The
|
||||
* buffer should be large enough to store the msgpack output (which
|
||||
@@ -17,7 +17,7 @@ using namespace simdjson;
|
||||
*
|
||||
* Recommended usage:
|
||||
*
|
||||
* simdjson2msgpack parser{};
|
||||
* simdjsonondemand2msgpack parser{};
|
||||
* simdjson::padded_string json = "[1,2]"_padded; // some JSON
|
||||
* uint8_t * buffer = new uint8_t[3*json.size() + simdjson::SIMDJSON_PADDING]; // large buffer
|
||||
*
|
||||
@@ -26,17 +26,17 @@ using namespace simdjson;
|
||||
* The result (msgpack) is a string view to a msgpack serialization of the input JSON,
|
||||
* it points inside the buffer you provided.
|
||||
*
|
||||
* You may reuse the simdjson2msgpack instance though you should use
|
||||
* You may reuse the simdjsonondemand2msgpack instance though you should use
|
||||
* one per thread.
|
||||
*/
|
||||
struct simdjson2msgpack {
|
||||
struct simdjsonondemand2msgpack {
|
||||
/**
|
||||
* @brief Converts the provided JSON into msgpack.
|
||||
*
|
||||
* @param json JSON input
|
||||
* @param buf temporary buffer (must be large enough, with simdjson::SIMDJSON_PADDING bytes
|
||||
* of padding)
|
||||
* @return std::string_view msgpack output, writting to the temporary buffer
|
||||
* @return std::string_view msgpack output, writing to the temporary buffer
|
||||
*/
|
||||
inline std::string_view to_msgpack(const simdjson::padded_string &json,
|
||||
uint8_t *buf);
|
||||
@@ -58,7 +58,7 @@ private:
|
||||
};
|
||||
|
||||
std::string_view
|
||||
simdjson2msgpack::to_msgpack(const simdjson::padded_string &json,
|
||||
simdjsonondemand2msgpack::to_msgpack(const simdjson::padded_string &json,
|
||||
uint8_t *buf) {
|
||||
buff = buf;
|
||||
ondemand::document doc = parser.iterate(json);
|
||||
@@ -106,33 +106,33 @@ simdjson2msgpack::to_msgpack(const simdjson::padded_string &json,
|
||||
return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
|
||||
}
|
||||
|
||||
void simdjson2msgpack::write_double(const double d) noexcept {
|
||||
void simdjsonondemand2msgpack::write_double(const double d) noexcept {
|
||||
*buff++ = 0xcb;
|
||||
::memcpy(buff, &d, sizeof(d));
|
||||
buff += sizeof(d);
|
||||
}
|
||||
|
||||
void simdjson2msgpack::write_byte(const uint8_t b) noexcept {
|
||||
void simdjsonondemand2msgpack::write_byte(const uint8_t b) noexcept {
|
||||
*buff = b;
|
||||
buff++;
|
||||
}
|
||||
|
||||
void simdjson2msgpack::write_uint32(const uint32_t w) noexcept {
|
||||
void simdjsonondemand2msgpack::write_uint32(const uint32_t w) noexcept {
|
||||
::memcpy(buff, &w, sizeof(w));
|
||||
buff += sizeof(w);
|
||||
}
|
||||
|
||||
uint8_t *simdjson2msgpack::skip_uint32() noexcept {
|
||||
uint8_t *simdjsonondemand2msgpack::skip_uint32() noexcept {
|
||||
uint8_t *ret = buff;
|
||||
buff += sizeof(uint32_t);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void simdjson2msgpack::write_uint32_at(const uint32_t w, uint8_t *p) noexcept {
|
||||
void simdjsonondemand2msgpack::write_uint32_at(const uint32_t w, uint8_t *p) noexcept {
|
||||
::memcpy(p, &w, sizeof(w));
|
||||
}
|
||||
|
||||
void simdjson2msgpack::write_raw_string(
|
||||
void simdjsonondemand2msgpack::write_raw_string(
|
||||
simdjson::ondemand::raw_json_string in) {
|
||||
write_byte(0xdb);
|
||||
uint8_t *location = skip_uint32();
|
||||
@@ -140,7 +140,7 @@ void simdjson2msgpack::write_raw_string(
|
||||
write_uint32_at(uint32_t(v.size()), location);
|
||||
}
|
||||
|
||||
void simdjson2msgpack::recursive_processor(simdjson::ondemand::value element) {
|
||||
void simdjsonondemand2msgpack::recursive_processor(simdjson::ondemand::value element) {
|
||||
switch (element.type()) {
|
||||
case simdjson::ondemand::json_type::array: {
|
||||
uint32_t counter = 0;
|
||||
@@ -185,7 +185,7 @@ void simdjson2msgpack::recursive_processor(simdjson::ondemand::value element) {
|
||||
}
|
||||
|
||||
|
||||
void simdjson2msgpack::recursive_processor_ref(simdjson::ondemand::value& element) {
|
||||
void simdjsonondemand2msgpack::recursive_processor_ref(simdjson::ondemand::value& element) {
|
||||
switch (element.type()) {
|
||||
case simdjson::ondemand::json_type::array: {
|
||||
uint32_t counter = 0;
|
||||
@@ -234,7 +234,7 @@ void simdjson2msgpack::recursive_processor_ref(simdjson::ondemand::value& elemen
|
||||
struct simdjson_ondemand {
|
||||
using StringType = std::string_view;
|
||||
|
||||
simdjson2msgpack parser{};
|
||||
simdjsonondemand2msgpack parser{};
|
||||
|
||||
bool run(simdjson::padded_string &json, char *buffer,
|
||||
std::string_view &result) {
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
#pragma once
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
|
||||
#include "json2msgpack.h"
|
||||
|
||||
namespace json2msgpack {
|
||||
|
||||
using namespace simdjson;
|
||||
|
||||
/**
|
||||
* @brief The simdjsonsinglestage2msgpack struct is used to quickly convert
|
||||
* JSON strings to msgpack views. You must provide a pointer to
|
||||
* a large memory region where the msgpack gets written. The
|
||||
* buffer should be large enough to store the msgpack output (which
|
||||
* can never be 3x larger than the input JSON) with an additional
|
||||
* simdjson::SIMDJSON_PADDING bytes.
|
||||
*
|
||||
* Recommended usage:
|
||||
*
|
||||
* simdjsonsinglestage2msgpack parser{};
|
||||
* simdjson::padded_string json = "[1,2]"_padded; // some JSON
|
||||
* uint8_t * buffer = new uint8_t[3*json.size() + simdjson::SIMDJSON_PADDING]; // large buffer
|
||||
*
|
||||
* std::string_view msgpack = parser.to_msgpack(json, buffer);
|
||||
*
|
||||
* The result (msgpack) is a string view to a msgpack serialization of the input JSON,
|
||||
* it points inside the buffer you provided.
|
||||
*
|
||||
* You may reuse the simdjsonsinglestage2msgpack instance though you should use
|
||||
* one per thread.
|
||||
*/
|
||||
struct simdjsonsinglestage2msgpack {
|
||||
/**
|
||||
* @brief Converts the provided JSON into msgpack.
|
||||
*
|
||||
* @param json JSON input
|
||||
* @param buf temporary buffer (must be large enough, with simdjson::SIMDJSON_PADDING bytes
|
||||
* of padding)
|
||||
* @return std::string_view msgpack output, writing to the temporary buffer
|
||||
*/
|
||||
inline std::string_view to_msgpack(const simdjson::padded_string &json,
|
||||
uint8_t *buf);
|
||||
|
||||
private:
|
||||
simdjson_inline void write_double(const double d) noexcept;
|
||||
simdjson_inline void write_byte(const uint8_t b) noexcept;
|
||||
simdjson_inline void write_uint32(const uint32_t w) noexcept;
|
||||
simdjson_inline uint8_t *skip_uint32() noexcept;
|
||||
simdjson_inline void write_uint32_at(const uint32_t w,
|
||||
uint8_t *p) noexcept;
|
||||
simdjson_inline void
|
||||
write_raw_string(simdjson::singlestage::raw_json_string rjs);
|
||||
inline void recursive_processor(simdjson::singlestage::value element);
|
||||
inline void recursive_processor_ref(simdjson::singlestage::value& element);
|
||||
|
||||
simdjson::singlestage::parser parser;
|
||||
uint8_t *buff{};
|
||||
};
|
||||
|
||||
std::string_view
|
||||
simdjsonsinglestage2msgpack::to_msgpack(const simdjson::padded_string &json,
|
||||
uint8_t *buf) {
|
||||
buff = buf;
|
||||
singlestage::document doc = parser.iterate(json);
|
||||
if (doc.is_scalar()) {
|
||||
// we have a special case where the JSON document is a single document...
|
||||
switch (doc.type()) {
|
||||
case simdjson::singlestage::json_type::number:
|
||||
write_double(doc.get_double());
|
||||
break;
|
||||
case simdjson::singlestage::json_type::string:
|
||||
write_raw_string(doc.get_raw_json_string());
|
||||
break;
|
||||
case simdjson::singlestage::json_type::boolean:
|
||||
write_byte(0xc2 + doc.get_bool());
|
||||
break;
|
||||
case simdjson::singlestage::json_type::null:
|
||||
// We check that the value is indeed null
|
||||
// otherwise: an error is thrown.
|
||||
if(doc.is_null()) {
|
||||
write_byte(0xc0);
|
||||
}
|
||||
break;
|
||||
case simdjson::singlestage::json_type::array:
|
||||
case simdjson::singlestage::json_type::object:
|
||||
default:
|
||||
// impossible
|
||||
SIMDJSON_UNREACHABLE();
|
||||
}
|
||||
} else {
|
||||
simdjson::singlestage::value val = doc;
|
||||
#define SIMDJSON_GCC_COMPILER ((__GNUC__) && !(__clang__) && !(__INTEL_COMPILER))
|
||||
#if SIMDJSON_GCC_COMPILER
|
||||
// the GCC compiler does well with by-value passing.
|
||||
// GCC has superior recursive inlining:
|
||||
// https://stackoverflow.com/questions/29186186/why-does-gcc-generate-a-faster-program-than-clang-in-this-recursive-fibonacci-co
|
||||
// https://godbolt.org/z/TeK4doE51
|
||||
recursive_processor(val);
|
||||
#else
|
||||
recursive_processor_ref(val);
|
||||
#endif
|
||||
}
|
||||
if (!doc.at_end()) {
|
||||
throw "There are unexpectedly tokens after the end of the json in the json2msgpack sample data";
|
||||
}
|
||||
return std::string_view(reinterpret_cast<char *>(buf), size_t(buff - buf));
|
||||
}
|
||||
|
||||
void simdjsonsinglestage2msgpack::write_double(const double d) noexcept {
|
||||
*buff++ = 0xcb;
|
||||
::memcpy(buff, &d, sizeof(d));
|
||||
buff += sizeof(d);
|
||||
}
|
||||
|
||||
void simdjsonsinglestage2msgpack::write_byte(const uint8_t b) noexcept {
|
||||
*buff = b;
|
||||
buff++;
|
||||
}
|
||||
|
||||
void simdjsonsinglestage2msgpack::write_uint32(const uint32_t w) noexcept {
|
||||
::memcpy(buff, &w, sizeof(w));
|
||||
buff += sizeof(w);
|
||||
}
|
||||
|
||||
uint8_t *simdjsonsinglestage2msgpack::skip_uint32() noexcept {
|
||||
uint8_t *ret = buff;
|
||||
buff += sizeof(uint32_t);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void simdjsonsinglestage2msgpack::write_uint32_at(const uint32_t w, uint8_t *p) noexcept {
|
||||
::memcpy(p, &w, sizeof(w));
|
||||
}
|
||||
|
||||
void simdjsonsinglestage2msgpack::write_raw_string(
|
||||
simdjson::singlestage::raw_json_string in) {
|
||||
write_byte(0xdb);
|
||||
uint8_t *location = skip_uint32();
|
||||
std::string_view v = parser.unescape(in, buff);
|
||||
write_uint32_at(uint32_t(v.size()), location);
|
||||
}
|
||||
|
||||
void simdjsonsinglestage2msgpack::recursive_processor(simdjson::singlestage::value element) {
|
||||
switch (element.type()) {
|
||||
case simdjson::singlestage::json_type::array: {
|
||||
uint32_t counter = 0;
|
||||
write_byte(0xdd);
|
||||
uint8_t *location = skip_uint32();
|
||||
for (auto child : element.get_array()) {
|
||||
counter++;
|
||||
recursive_processor(child.value());
|
||||
}
|
||||
write_uint32_at(counter, location);
|
||||
} break;
|
||||
case simdjson::singlestage::json_type::object: {
|
||||
uint32_t counter = 0;
|
||||
write_byte(0xdf);
|
||||
uint8_t *location = skip_uint32();
|
||||
for (auto field : element.get_object()) {
|
||||
counter++;
|
||||
write_raw_string(field.key());
|
||||
recursive_processor(field.value());
|
||||
}
|
||||
write_uint32_at(counter, location);
|
||||
} break;
|
||||
case simdjson::singlestage::json_type::number:
|
||||
write_double(element.get_double());
|
||||
break;
|
||||
case simdjson::singlestage::json_type::string:
|
||||
write_raw_string(element.get_raw_json_string());
|
||||
break;
|
||||
case simdjson::singlestage::json_type::boolean:
|
||||
write_byte(0xc2 + element.get_bool());
|
||||
break;
|
||||
case simdjson::singlestage::json_type::null:
|
||||
// We check that the value is indeed null
|
||||
// otherwise: an error is thrown.
|
||||
if(element.is_null()) {
|
||||
write_byte(0xc0);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
SIMDJSON_UNREACHABLE();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void simdjsonsinglestage2msgpack::recursive_processor_ref(simdjson::singlestage::value& element) {
|
||||
switch (element.type()) {
|
||||
case simdjson::singlestage::json_type::array: {
|
||||
uint32_t counter = 0;
|
||||
write_byte(0xdd);
|
||||
uint8_t *location = skip_uint32();
|
||||
for (auto child : element.get_array()) {
|
||||
counter++;
|
||||
simdjson::singlestage::value v = child.value();
|
||||
recursive_processor_ref(v);
|
||||
}
|
||||
write_uint32_at(counter, location);
|
||||
} break;
|
||||
case simdjson::singlestage::json_type::object: {
|
||||
uint32_t counter = 0;
|
||||
write_byte(0xdf);
|
||||
uint8_t *location = skip_uint32();
|
||||
for (auto field : element.get_object()) {
|
||||
counter++;
|
||||
write_raw_string(field.key());
|
||||
simdjson::singlestage::value v = field.value();
|
||||
recursive_processor_ref(v);
|
||||
}
|
||||
write_uint32_at(counter, location);
|
||||
} break;
|
||||
case simdjson::singlestage::json_type::number:
|
||||
write_double(element.get_double());
|
||||
break;
|
||||
case simdjson::singlestage::json_type::string:
|
||||
write_raw_string(element.get_raw_json_string());
|
||||
break;
|
||||
case simdjson::singlestage::json_type::boolean:
|
||||
write_byte(0xc2 + element.get_bool());
|
||||
break;
|
||||
case simdjson::singlestage::json_type::null:
|
||||
// We check that the value is indeed null
|
||||
// otherwise: an error is thrown.
|
||||
if(element.is_null()) {
|
||||
write_byte(0xc0);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
SIMDJSON_UNREACHABLE();
|
||||
}
|
||||
}
|
||||
|
||||
struct simdjson_singlestage {
|
||||
using StringType = std::string_view;
|
||||
|
||||
simdjsonsinglestage2msgpack parser{};
|
||||
|
||||
bool run(simdjson::padded_string &json, char *buffer,
|
||||
std::string_view &result) {
|
||||
result = parser.to_msgpack(json, reinterpret_cast<uint8_t *>(buffer));
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
BENCHMARK_TEMPLATE(json2msgpack, simdjson_singlestage)->UseManualTime();
|
||||
|
||||
} // namespace json2msgpack
|
||||
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
@@ -13,6 +13,8 @@ void maybe_display_implementation() {
|
||||
std::cout << "simdjson::dom implementation: " << simdjson::get_active_implementation()->name() << std::endl;
|
||||
std::cout << "simdjson::ondemand implementation (stage 1): " << simdjson::get_active_implementation()->name() << std::endl;
|
||||
std::cout << "simdjson::ondemand implementation (stage 2): " << simdjson::builtin_implementation()->name() << std::endl;
|
||||
std::cout << "simdjson::singlestage implementation (stage 1): " << simdjson::get_active_implementation()->name() << std::endl;
|
||||
std::cout << "simdjson::singlestage implementation (stage 2): " << simdjson::builtin_implementation()->name() << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
#pragma once
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
|
||||
#include "kostya.h"
|
||||
|
||||
namespace kostya {
|
||||
|
||||
using namespace simdjson;
|
||||
|
||||
struct simdjson_singlestage {
|
||||
static constexpr diff_flags DiffFlags = diff_flags::NONE;
|
||||
|
||||
singlestage::parser parser{};
|
||||
|
||||
bool run(simdjson::padded_string &json, std::vector<point> &result) {
|
||||
auto doc = parser.iterate(json);
|
||||
for (singlestage::object point : doc.find_field("coordinates")) {
|
||||
result.emplace_back(json_benchmark::point{point.find_field("x"), point.find_field("y"), point.find_field("z")});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
BENCHMARK_TEMPLATE(kostya, simdjson_singlestage)->UseManualTime();
|
||||
|
||||
} // namespace kostya
|
||||
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
@@ -0,0 +1,72 @@
|
||||
#pragma once
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
|
||||
#include "large_amazon_cellphones.h"
|
||||
|
||||
namespace large_amazon_cellphones {
|
||||
|
||||
using namespace simdjson;
|
||||
|
||||
template<bool threaded>
|
||||
struct simdjson_singlestage {
|
||||
using StringType = std::string;
|
||||
|
||||
singlestage::parser parser{};
|
||||
|
||||
bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
|
||||
#ifdef SIMDJSON_THREADS_ENABLED
|
||||
parser.threaded = threaded;
|
||||
#endif
|
||||
singlestage::document_stream stream = parser.iterate_many(json);
|
||||
singlestage::document_stream::iterator i = stream.begin();
|
||||
++i; // Skip first line
|
||||
for (;i != stream.end(); ++i) {
|
||||
auto doc = *i;
|
||||
size_t index{0};
|
||||
StringType copy;
|
||||
double rating;
|
||||
uint64_t reviews;
|
||||
for ( auto value : doc ) {
|
||||
switch (index)
|
||||
{
|
||||
case 1:
|
||||
copy = StringType(std::string_view(value));
|
||||
break;
|
||||
case 5:
|
||||
rating = double(value);
|
||||
break;
|
||||
case 7:
|
||||
reviews = uint64_t(value);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
index++;
|
||||
}
|
||||
|
||||
auto x = result.find(copy);
|
||||
if (x == result.end()) { // If key not found, add new key
|
||||
result.emplace(copy, large_amazon_cellphones::brand{
|
||||
rating * reviews,
|
||||
reviews
|
||||
});
|
||||
} else { // Otherwise, update key data
|
||||
x->second.cumulative_rating += rating * reviews;
|
||||
x->second.reviews_count += reviews;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_singlestage<UNTHREADED>)->UseManualTime();
|
||||
#ifdef SIMDJSON_THREADS_ENABLED
|
||||
BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_singlestage<THREADED>)->UseManualTime();
|
||||
#endif
|
||||
|
||||
} // namespace amazon_cellphones
|
||||
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
@@ -0,0 +1,29 @@
|
||||
#pragma once
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
|
||||
#include "large_random.h"
|
||||
|
||||
namespace large_random {
|
||||
|
||||
using namespace simdjson;
|
||||
|
||||
struct simdjson_singlestage {
|
||||
static constexpr diff_flags DiffFlags = diff_flags::NONE;
|
||||
|
||||
singlestage::parser parser{};
|
||||
|
||||
bool run(simdjson::padded_string &json, std::vector<point> &result) {
|
||||
auto doc = parser.iterate(json);
|
||||
for (singlestage::object coord : doc) {
|
||||
result.emplace_back(json_benchmark::point{coord.find_field("x"), coord.find_field("y"), coord.find_field("z")});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
BENCHMARK_TEMPLATE(large_random, simdjson_singlestage)->UseManualTime();
|
||||
|
||||
} // namespace large_random
|
||||
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
@@ -0,0 +1,29 @@
|
||||
#pragma once
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
|
||||
#include "large_random.h"
|
||||
|
||||
namespace large_random {
|
||||
|
||||
using namespace simdjson;
|
||||
|
||||
struct simdjson_singlestage_unordered {
|
||||
static constexpr diff_flags DiffFlags = diff_flags::NONE;
|
||||
|
||||
singlestage::parser parser{};
|
||||
|
||||
bool run(simdjson::padded_string &json, std::vector<point> &result) {
|
||||
auto doc = parser.iterate(json);
|
||||
for (singlestage::object coord : doc) {
|
||||
result.emplace_back(json_benchmark::point{coord["x"], coord["y"], coord["z"]});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
BENCHMARK_TEMPLATE(large_random, simdjson_singlestage_unordered)->UseManualTime();
|
||||
|
||||
} // namespace large_random
|
||||
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
@@ -0,0 +1,37 @@
|
||||
#pragma once
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
|
||||
#include "largerandom.h"
|
||||
|
||||
namespace largerandom {
|
||||
|
||||
using namespace simdjson;
|
||||
|
||||
class OnDemand {
|
||||
public:
|
||||
simdjson_inline bool Run(const padded_string &json);
|
||||
simdjson_inline const std::vector<my_point> &Result() { return container; }
|
||||
simdjson_inline size_t ItemCount() { return container.size(); }
|
||||
|
||||
private:
|
||||
singlestage::parser parser{};
|
||||
std::vector<my_point> container{};
|
||||
};
|
||||
|
||||
simdjson_inline bool OnDemand::Run(const padded_string &json) {
|
||||
container.clear();
|
||||
|
||||
auto doc = parser.iterate(json);
|
||||
for (singlestage::object coord : doc) {
|
||||
container.emplace_back(my_point{coord.find_field("x"), coord.find_field("y"), coord.find_field("z")});
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
BENCHMARK_TEMPLATE(LargeRandom, OnDemand);
|
||||
|
||||
} // namespace largerandom
|
||||
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
@@ -0,0 +1,48 @@
|
||||
#pragma once
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
|
||||
#include "partial_tweets.h"
|
||||
|
||||
namespace partial_tweets {
|
||||
|
||||
using namespace simdjson;
|
||||
|
||||
struct simdjson_singlestage {
|
||||
using StringType=std::string_view;
|
||||
|
||||
singlestage::parser parser{};
|
||||
|
||||
simdjson_inline uint64_t nullable_int(singlestage::value value) {
|
||||
if (value.is_null()) { return 0; }
|
||||
return value;
|
||||
}
|
||||
|
||||
simdjson_inline twitter_user<std::string_view> read_user(singlestage::object user) {
|
||||
return { user.find_field("id"), user.find_field("screen_name") };
|
||||
}
|
||||
|
||||
bool run(simdjson::padded_string &json, std::vector<tweet<std::string_view>> &result) {
|
||||
// Walk the document, parsing the tweets as we go
|
||||
auto doc = parser.iterate(json);
|
||||
for (singlestage::object tweet : doc.find_field("statuses")) {
|
||||
result.emplace_back(partial_tweets::tweet<std::string_view>{
|
||||
tweet.find_field("created_at"),
|
||||
tweet.find_field("id"),
|
||||
tweet.find_field("text"),
|
||||
nullable_int(tweet.find_field("in_reply_to_status_id")),
|
||||
read_user(tweet.find_field("user")),
|
||||
tweet.find_field("retweet_count"),
|
||||
tweet.find_field("favorite_count")
|
||||
});
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
BENCHMARK_TEMPLATE(partial_tweets, simdjson_singlestage)->UseManualTime();
|
||||
|
||||
} // namespace partial_tweets
|
||||
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
@@ -0,0 +1,63 @@
|
||||
#pragma once
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
|
||||
#include "partial_tweets.h"
|
||||
|
||||
namespace partial_tweets {
|
||||
|
||||
using namespace simdjson;
|
||||
|
||||
|
||||
class SingleStage {
|
||||
public:
|
||||
SingleStage() {
|
||||
if(!displayed_implementation) {
|
||||
std::cout << "On Demand implementation: " << builtin_implementation()->name() << std::endl;
|
||||
displayed_implementation = true;
|
||||
}
|
||||
}
|
||||
simdjson_inline bool Run(const padded_string &json);
|
||||
simdjson_inline const std::vector<tweet> &Result() { return tweets; }
|
||||
simdjson_inline size_t ItemCount() { return tweets.size(); }
|
||||
|
||||
private:
|
||||
singlestage::parser parser{};
|
||||
std::vector<tweet> tweets{};
|
||||
|
||||
simdjson_inline uint64_t nullable_int(singlestage::value value) {
|
||||
if (value.is_null()) { return 0; }
|
||||
return value;
|
||||
}
|
||||
|
||||
simdjson_inline twitter_user read_user(singlestage::object user) {
|
||||
return { user.find_field("id"), user.find_field("screen_name") };
|
||||
}
|
||||
|
||||
static inline bool displayed_implementation = false;
|
||||
};
|
||||
|
||||
simdjson_inline bool SingleStage::Run(const padded_string &json) {
|
||||
tweets.clear();
|
||||
|
||||
// Walk the document, parsing the tweets as we go
|
||||
auto doc = parser.iterate(json);
|
||||
for (singlestage::object tweet : doc.find_field("statuses")) {
|
||||
tweets.emplace_back(partial_tweets::tweet{
|
||||
tweet.find_field("created_at"),
|
||||
tweet.find_field("id"),
|
||||
tweet.find_field("text"),
|
||||
nullable_int(tweet.find_field("in_reply_to_status_id")),
|
||||
read_user(tweet.find_field("user")),
|
||||
tweet.find_field("retweet_count"),
|
||||
tweet.find_field("favorite_count")
|
||||
});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
BENCHMARK_TEMPLATE(PartialTweets, SingleStage);
|
||||
|
||||
} // namespace partial_tweets
|
||||
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
@@ -0,0 +1,80 @@
|
||||
#pragma once
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
|
||||
#include "top_tweet.h"
|
||||
|
||||
namespace top_tweet {
|
||||
|
||||
using namespace simdjson;
|
||||
|
||||
struct simdjson_singlestage {
|
||||
using StringType=std::string_view;
|
||||
|
||||
singlestage::parser parser{};
|
||||
|
||||
bool run(simdjson::padded_string &json, int64_t max_retweet_count, top_tweet_result<StringType> &result) {
|
||||
result.retweet_count = -1;
|
||||
// We save these DOM values for later so we don't have to parse them
|
||||
// into string_views until we're sure which ones we want to parse
|
||||
// NOTE: simdjson does not presently support reuse of objects or arrays--just scalars. This is
|
||||
// why we have to grab the text and screen_name fields instead of just saving the tweet object.
|
||||
singlestage::value screen_name, text;
|
||||
|
||||
auto doc = parser.iterate(json);
|
||||
for (auto tweet : doc["statuses"]) {
|
||||
// Since text, user.screen_name, and retweet_count generally appear in order, it's nearly free
|
||||
// for us to retrieve them here (and will cost a bit more if we do it in the if
|
||||
// statement).
|
||||
auto tweet_text = tweet["text"];
|
||||
auto tweet_screen_name = tweet["user"]["screen_name"];
|
||||
int64_t retweet_count = tweet["retweet_count"];
|
||||
if (retweet_count <= max_retweet_count && retweet_count >= result.retweet_count) {
|
||||
result.retweet_count = retweet_count;
|
||||
// TODO std::move should not be necessary
|
||||
text = std::move(tweet_text);
|
||||
screen_name = std::move(tweet_screen_name);
|
||||
}
|
||||
}
|
||||
|
||||
// Now that we know which was the most retweeted, parse the values in it
|
||||
result.screen_name = screen_name;
|
||||
result.text = text;
|
||||
return result.retweet_count != -1;
|
||||
}
|
||||
};
|
||||
|
||||
BENCHMARK_TEMPLATE(top_tweet, simdjson_singlestage)->UseManualTime();
|
||||
|
||||
struct simdjson_singlestage_forward_only {
|
||||
using StringType=std::string_view;
|
||||
|
||||
singlestage::parser parser{};
|
||||
|
||||
bool run(simdjson::padded_string &json, int64_t max_retweet_count, top_tweet_result<StringType> &result) {
|
||||
result.retweet_count = -1;
|
||||
|
||||
auto doc = parser.iterate(json);
|
||||
for (auto tweet : doc["statuses"]) {
|
||||
// Since text, user.screen_name, and retweet_count generally appear in order, it's nearly free
|
||||
// for us to retrieve them here (and will cost a bit more if we do it in the if
|
||||
// statement).
|
||||
auto tweet_text = tweet["text"];
|
||||
auto tweet_screen_name = tweet["user"]["screen_name"];
|
||||
int64_t retweet_count = tweet["retweet_count"];
|
||||
if (retweet_count <= max_retweet_count && retweet_count >= result.retweet_count) {
|
||||
result.retweet_count = retweet_count;
|
||||
result.text = tweet_text;
|
||||
result.screen_name = tweet_screen_name;
|
||||
}
|
||||
}
|
||||
|
||||
return result.retweet_count != -1;
|
||||
}
|
||||
};
|
||||
|
||||
BENCHMARK_TEMPLATE(top_tweet, simdjson_singlestage_forward_only)->UseManualTime();
|
||||
|
||||
} // namespace top_tweet
|
||||
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
+16
-8
@@ -335,8 +335,8 @@ support for users who avoid exceptions. See [the simdjson error handling documen
|
||||
`double(element)`. This works for `std::string_view`, double, uint64_t, int64_t, bool,
|
||||
ondemand::object and ondemand::array. We also have explicit methods such as `get_string()`, `get_double()`,
|
||||
`get_uint64()`, `get_int64()`, `get_bool()`, `get_object()` and `get_array()`. After a cast or an explicit method,
|
||||
the number, string or boolean will be parsed, or the initial `{` or `[` will be verified for ondemand::object and ondemand::array. An exception is thrown if
|
||||
the cast is not possible. Importantly, when getting an ondemand::object or ondemand::array instance, its content is
|
||||
the number, string or boolean will be parsed, or the initial `{` or `[` will be verified for `ondemand::object` and `ondemand::array`. An exception may be thrown if
|
||||
the cast is not possible: there error code is `simdjson::INCORRECT_TYPE` (see [Error Handling](#error-handling)). Importantly, when getting an ondemand::object or ondemand::array instance, its content is
|
||||
not validated: you are only guaranteed that the corresponding initial character (`{` or `[`) is present. Thus,
|
||||
for example, you could have an ondemand::object instance pointing at the invalid JSON `{ "this is not a valid object" }`: the validation occurs as you access the content.
|
||||
The `get_string()` returns a valid UTF-8 string, after
|
||||
@@ -354,8 +354,8 @@ support for users who avoid exceptions. See [the simdjson error handling documen
|
||||
> `get_string()` twice on an object (or to cast an object twice to `std::string_view`).
|
||||
* **Field Access:** To get the value of the "foo" field in an object, use `object["foo"]`. This will
|
||||
scan through the object looking for the field with the matching string, doing a character-by-character
|
||||
comparison. For efficiency reason, you should avoid looking up the same field repeatedly: e.g., do
|
||||
not do `object["foo"]` followed by `object["foo"]` with the same `object` instance. Keep in mind that On Demand does not buffer or save the result of the parsing: if you repeatedly access `object["foo"]`, then it must repeatedly seek the key and parse the content. The library does not provide a distinct function to check if a key is present, instead we recommend you attempt to access the key: e.g., by doing `ondemand::value val{}; if(!object["foo"].get(val)) {...}`, you have that `val` contains the requested value inside the if clause. It is your responsability as a user to temporarily keep a reference to the value (`auto v = object["foo"]`), or to consume the content and store it in your own data structures. If you consume an
|
||||
comparison. It may generate the error `simdjson::NO_SUCH_FIELD` if there is no such key in the object, it may throw an exception (see [Error Handling](#error-handling)). For efficiency reason, you should avoid looking up the same field repeatedly: e.g., do
|
||||
not do `object["foo"]` followed by `object["foo"]` with the same `object` instance. Keep in mind that On Demand does not buffer or save the result of the parsing: if you repeatedly access `object["foo"]`, then it must repeatedly seek the key and parse the content. The library does not provide a distinct function to check if a key is present, instead we recommend you attempt to access the key: e.g., by doing `ondemand::value val{}; if(!object["foo"].get(val)) {...}`, you have that `val` contains the requested value inside the if clause. It is your responsibility as a user to temporarily keep a reference to the value (`auto v = object["foo"]`), or to consume the content and store it in your own data structures. If you consume an
|
||||
object twice: `std::string_view(object["foo"]` followed by `std::string_view(object["foo"]` then your code
|
||||
is in error. Furthermore, you can only consume one field at a time, on the same object. The
|
||||
value instance you get from `content["bids"]` becomes invalid when you call `content["asks"]`.
|
||||
@@ -887,10 +887,17 @@ auto error = parser.iterate(json).get(doc);
|
||||
if (error) { cerr << error << endl; exit(1); }
|
||||
```
|
||||
|
||||
When there is no error, the error code simdjson::SUCCESS is returned: it evaluates as false as a Boolean.
|
||||
When there is no error, the error code `simdjson::SUCCESS`is returned: it evaluates as false as a Boolean.
|
||||
We have several error codes to indicate errors, they all evaluate to true as a Boolean: your software should not generally not depend on exact
|
||||
error codes. We may change the error codes in future releases and the exact error codes could vary depending on your system.
|
||||
|
||||
Some errors are recoverable:
|
||||
|
||||
* You may get the error `simdjson::INCORRECT_TYPE` after trying to convert a value to an incorrect type: e.g., you expected a number and try to convert the value to a number, but it is an array.
|
||||
* You may query a key from an object, but the key is missing in which case you get the error `simdjson::NO_SUCH_FIELD`: e.g., you call `obj["myname"]` and the object does not have a key `"myname"`.
|
||||
|
||||
Other errors (e.g., `simdjson::INCOMPLETE_ARRAY_OR_OBJECT`) may indicate a fatal error and often follow from the fact that the document is not valid JSON. In which case, it is no longer possible to continue accessing the document: calling the method `is_alive()` on the document instance returns false. All following accesses will keep returning the same fatal error (e.g., `simdjson::INCOMPLETE_ARRAY_OR_OBJECT`).
|
||||
|
||||
When you use the code without exceptions, it is your responsibility to check for error before using the
|
||||
result: if there is an error, the result value will not be valid and using it will caused undefined behavior. Most compilers should be able to help you if you activate the right
|
||||
set of warnings: they can identify variables that are written to but never otherwise accessed.
|
||||
@@ -1422,10 +1429,9 @@ This parser can't support a document that big
|
||||
|
||||
If your documents are large (e.g., larger than a megabyte), then the `iterate_many` function is maybe ill-suited. It is really meant to support reading efficiently streams of relatively small documents (e.g., a few kilobytes each). If you have larger documents, you should use other functions like `iterate`.
|
||||
|
||||
We also provide some support for comma-separated documents and other advanced features.
|
||||
See [iterate_many.md](iterate_many.md) for detailed information and design.
|
||||
|
||||
|
||||
|
||||
Parsing Numbers Inside Strings
|
||||
------------------------------
|
||||
|
||||
@@ -1948,6 +1954,8 @@ Performance Tips
|
||||
- Given a field `field` in an object, calling `field.key()` is often faster than `field.unescaped_key()` so if you do not need an unescaped `std::string_view` instance, prefer `field.key()`.
|
||||
- For release builds, we recommend setting `NDEBUG` pre-processor directive when compiling the `simdjson` library. Importantly, using the optimization flags `-O2` or `-O3` under GCC and LLVM clang does not set the `NDEBUG` directive, you must set it manually (e.g., `-DNDEBUG`).
|
||||
- For long streams of JSON documents, consider [`iterate_many`](iterate_many.md) and [`parse_many`](parse_many.md) for better performance.
|
||||
- Never seek to access a field twice (e.g., o["data"] and later again o["data"]). Instead capture once an ondemand::value and reuse it.
|
||||
- If you must access several different keys in an object, it might be preferable to iterate through all the fields in the object instead, and branch on the field keys.
|
||||
- If possible, refer to each object and array in your code once. For example, the following code repeatedly refers to the `"data"` key to create an object...
|
||||
```C++
|
||||
std::string_view make = o["data"]["make"];
|
||||
@@ -1961,4 +1969,4 @@ Performance Tips
|
||||
std::string_view year = data["year"];
|
||||
std::string_view rating = data["rating"];
|
||||
```
|
||||
- To better understand the operation of your On Demand parser, and whether it is performing as well as you think it should be, there is a logger feature built in to simdjson! To use it, define the pre-processor directive `SIMDJSON_VERBOSE_LOGGING` prior to including the `simdjson.h` header, which enables logging in simdjson. Run your code. It may generate a lot of logging output; adding printouts from your application that show each section may be helpful. The log’s output will show step-by-step information on state, buffer pointer position, depth, and key retrieval status. The log level is set to `INFO` by default, but can be modified with the environment variable `SIMDJSON_LOG_LEVEL`. Setting the environment variable `SIMDJSON_LOG_LEVEL` to `ERROR` will log only errors.
|
||||
- To better understand the operation of your On Demand parser, and whether it is performing as well as you think it should be, there is a logger feature built in to simdjson! To use it, define the pre-processor directive `SIMDJSON_VERBOSE_LOGGING` prior to including the `simdjson.h` header, which enables logging in simdjson. Run your code. It may generate a lot of logging output; adding printouts from your application that show each section may be helpful. The log's output will show step-by-step information on state, buffer pointer position, depth, and key retrieval status. Importantly, unless `SIMDJSON_VERBOSE_LOGGING` is defined, logging is entirely disabled and thus carries no overhead.
|
||||
+1
-1
@@ -105,7 +105,7 @@ Once you have an element, you can navigate it with idiomatic C++ iterators, oper
|
||||
with the `size()` method.
|
||||
* **Checking an Element Type:** You can check an element's type with `element.type()`. It
|
||||
returns an `element_type` with values such as `simdjson::dom::element_type::ARRAY`, `simdjson::dom::element_type::OBJECT`, `simdjson::dom::element_type::INT64`, `simdjson::dom::element_type::UINT64`,`simdjson::dom::element_type::DOUBLE`, `simdjson::dom::element_type::STRING`, `simdjson::dom::element_type::BOOL` or, `simdjson::dom::element_type::NULL_VALUE`.
|
||||
* **Output to streams and strings:** Given a document or an element (or node) out of a JSON document, you can output a minified string version using the C++ stream idiom (`out << element`). You can also request the construction of a minified string version (`simdjson::minify(element)`). Numbers are serialized as 64-bit floating-point numbers (`double`).
|
||||
* **Output to streams and strings:** Given a document or an element (or node) out of a JSON document, you can output a minified string version using the C++ stream idiom (`out << element`). You can also request the construction of a minified string version (`simdjson::minify(element)`) or a prettified string version (`simdjson::prettify(element)`). Numbers are serialized as 64-bit floating-point numbers (`double`).
|
||||
|
||||
### Examples
|
||||
|
||||
|
||||
+45
-2
@@ -129,6 +129,18 @@ Some official formats **(non-exhaustive list)**:
|
||||
API
|
||||
---
|
||||
|
||||
Example:
|
||||
|
||||
```c++
|
||||
auto json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )"_padded;
|
||||
ondemand::parser parser;
|
||||
ondemand::document_stream docs = parser.iterate_many(json);
|
||||
for (auto doc : docs) {
|
||||
std::cout << doc["foo"] << std::endl;
|
||||
}
|
||||
// Prints 1 2 3
|
||||
```
|
||||
|
||||
See [basics.md](basics.md#newline-delimited-json-ndjson-and-json-lines) for an overview of the API.
|
||||
|
||||
## Use cases
|
||||
@@ -238,7 +250,38 @@ This will print:
|
||||
|
||||
Importantly, you should only call `truncated_bytes()` after iterating through all of the documents since the stream cannot tell whether there are truncated documents at the very end when it may not have accessed that part of the data yet.
|
||||
|
||||
Comma separated documents
|
||||
Comma-separated documents
|
||||
-----------
|
||||
|
||||
`iterate_many` also takes in an option to allow parsing of comma separated documents. In this mode, the entire buffer is processed in 1 batch and batch size will be increased to be as large as the JSON passed. Therefore, the capacity of the parser has to be sufficient to support the batch size set.
|
||||
We also support comma-separated documents, but with some performance limitations. The `iterate_many` function takes in an option to allow parsing of comma separated documents (which defaults on false). In this mode, the entire buffer is processed in one batch. Therefore, the total size of the document should not exceed the maximal capacity of the parser (4 GB). This mode also effectively disallow multithreading. It is therefore mostly suitable for not "very large" inputs. In this mode, the batch_size parameter
|
||||
is effectively ignored, as it is set to at least the document size.
|
||||
|
||||
Example:
|
||||
|
||||
```C++
|
||||
auto json = R"( 1, 2, 3, 4, "a", "b", "c", {"hello": "world"} , [1, 2, 3])"_padded;
|
||||
ondemand::parser parser;
|
||||
ondemand::document_stream doc_stream;
|
||||
// We pass '32' as the batch size, but it is a bogus parameter because, since
|
||||
// we pass 'true' to the allow_comma parameter, the batch size will be set to at least
|
||||
// the document size.
|
||||
auto error = parser.iterate_many(json, 32, true).get(doc_stream);
|
||||
if(error) { std::cerr << error << std::endl; return; }
|
||||
for (auto doc : doc_stream) {
|
||||
std::cout << doc.type() << std::endl;
|
||||
}
|
||||
```
|
||||
|
||||
This will print:
|
||||
|
||||
```
|
||||
number
|
||||
number
|
||||
number
|
||||
number
|
||||
string
|
||||
string
|
||||
string
|
||||
object
|
||||
array
|
||||
```
|
||||
|
||||
@@ -35,4 +35,14 @@ IF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
|
||||
add_quickstart_test(quickstart_ondemand_noexceptions quickstart_ondemand_noexceptions.cpp NO_EXCEPTIONS LABELS quickstart_ondemand acceptance)
|
||||
add_quickstart_test(quickstart_ondemand_noexceptions11 quickstart_ondemand_noexceptions.cpp NO_EXCEPTIONS CXX_STANDARD c++11 LABELS quickstart_ondemand)
|
||||
|
||||
# SingleStage Quick Start
|
||||
if (SIMDJSON_EXCEPTIONS)
|
||||
add_quickstart_test(quickstart_singlestage quickstart_singlestage.cpp LABELS quickstart_singlestage acceptance)
|
||||
add_quickstart_test(quickstart_singlestage11 quickstart_singlestage.cpp CXX_STANDARD c++11 LABELS quickstart_singlestage acceptance)
|
||||
add_quickstart_test(quickstart_singlestage14 quickstart_singlestage.cpp CXX_STANDARD c++14 LABELS quickstart_singlestage)
|
||||
endif()
|
||||
|
||||
add_quickstart_test(quickstart_singlestage_noexceptions quickstart_singlestage_noexceptions.cpp NO_EXCEPTIONS LABELS quickstart_singlestage acceptance)
|
||||
add_quickstart_test(quickstart_singlestage_noexceptions11 quickstart_singlestage_noexceptions.cpp NO_EXCEPTIONS CXX_STANDARD c++11 LABELS quickstart_singlestage)
|
||||
|
||||
endif()
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
#include <iostream>
|
||||
#include "simdjson.h"
|
||||
using namespace simdjson;
|
||||
int main(void) {
|
||||
ondemand::parser parser;
|
||||
padded_string json = padded_string::load("twitter.json");
|
||||
ondemand::document tweets = parser.iterate(json);
|
||||
std::cout << uint64_t(tweets["search_metadata"]["count"]) << " results." << std::endl;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
#include <iostream>
|
||||
#include "simdjson.h"
|
||||
using namespace simdjson;
|
||||
int main(void) {
|
||||
padded_string json;
|
||||
auto error = padded_string::load("twitter.json").get(json);
|
||||
if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
|
||||
|
||||
ondemand::parser parser;
|
||||
ondemand::document tweets;
|
||||
error = parser.iterate(json).get(tweets);
|
||||
if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
|
||||
|
||||
uint64_t count;
|
||||
error = tweets["search_metadata"]["count"].get(count);
|
||||
if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
|
||||
|
||||
std::cout << count << " results." << std::endl;
|
||||
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
@@ -63,6 +63,7 @@ if(SIMDJSON_ENABLE_FUZZING)
|
||||
implement_fuzzer(fuzz_minifyimpl) # minify *without* parsing, plus compare implementations
|
||||
implement_fuzzer(fuzz_ndjson) # the ndjson api
|
||||
implement_fuzzer(fuzz_ondemand)
|
||||
# implement_fuzzer(fuzz_singlestage) # TODO why no worky
|
||||
implement_fuzzer(fuzz_padded)
|
||||
implement_fuzzer(fuzz_parser)
|
||||
implement_fuzzer(fuzz_print_json)
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
#include "FuzzUtils.h"
|
||||
#include "simdjson.h"
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
|
||||
FuzzData fd(Data, Size);
|
||||
const int action = fd.getInt<0, 12>();
|
||||
|
||||
// split the remainder of the document into strings
|
||||
auto strings = fd.splitIntoStrings();
|
||||
while (strings.size() < 1) {
|
||||
strings.emplace_back();
|
||||
}
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
try {
|
||||
#endif
|
||||
simdjson::singlestage::parser parser;
|
||||
simdjson::padded_string padded(strings[0]);
|
||||
auto doc = parser.iterate(padded);
|
||||
if (doc.error()) {
|
||||
return 0;
|
||||
}
|
||||
for (auto item : doc) {
|
||||
switch (action) {
|
||||
case 0: {
|
||||
simdjson_unused auto x = item.get_string();
|
||||
} break;
|
||||
case 1: {
|
||||
simdjson_unused auto x = item.get_bool();
|
||||
} break;
|
||||
case 2: {
|
||||
simdjson_unused auto x = item.get_array();
|
||||
} break;
|
||||
case 3: {
|
||||
simdjson_unused auto x = item.get_int64();
|
||||
} break;
|
||||
case 4: {
|
||||
simdjson_unused auto x = item.get_double();
|
||||
} break;
|
||||
case 5: {
|
||||
simdjson_unused auto x = item.get_object();
|
||||
} break;
|
||||
case 6: {
|
||||
simdjson_unused auto x = item.get_uint64();
|
||||
} break;
|
||||
case 7: {
|
||||
simdjson_unused auto x = item.get_raw_json_string();
|
||||
} break;
|
||||
case 8: {
|
||||
simdjson_unused auto x = item.is_null();
|
||||
} break;
|
||||
case 9: {
|
||||
simdjson_unused auto x = item.begin();
|
||||
} break;
|
||||
case 10: {
|
||||
simdjson_unused auto x = item.end();
|
||||
} break;
|
||||
case 11: {
|
||||
for (auto e : item) {
|
||||
simdjson_unused auto x = e.is_null();
|
||||
}
|
||||
} break;
|
||||
default:;
|
||||
}
|
||||
}
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
} catch (...) {
|
||||
}
|
||||
#endif
|
||||
|
||||
return 0;
|
||||
}
|
||||
+18
-1
@@ -34,8 +34,25 @@
|
||||
}
|
||||
*/
|
||||
|
||||
#include "simdjson/common_defs.h"
|
||||
|
||||
// This provides the public API for simdjson.
|
||||
// DOM and ondemand are amalgamated separately, in simdjson.h
|
||||
#include "simdjson/simdjson_version.h"
|
||||
|
||||
#include "simdjson/base.h"
|
||||
|
||||
#include "simdjson/error.h"
|
||||
#include "simdjson/error-inl.h"
|
||||
#include "simdjson/implementation.h"
|
||||
#include "simdjson/minify.h"
|
||||
#include "simdjson/padded_string.h"
|
||||
#include "simdjson/padded_string-inl.h"
|
||||
#include "simdjson/padded_string_view.h"
|
||||
#include "simdjson/padded_string_view-inl.h"
|
||||
|
||||
#include "simdjson/dom.h"
|
||||
#include "simdjson/builtin.h"
|
||||
#include "simdjson/ondemand.h"
|
||||
#include "simdjson/singlestage.h"
|
||||
|
||||
#endif // SIMDJSON_H
|
||||
|
||||
@@ -1,34 +1,8 @@
|
||||
#ifndef SIMDJSON_ARM64_H
|
||||
#define SIMDJSON_ARM64_H
|
||||
|
||||
#include "simdjson/implementation-base.h"
|
||||
|
||||
#if SIMDJSON_IMPLEMENTATION_ARM64
|
||||
|
||||
namespace simdjson {
|
||||
/**
|
||||
* Implementation for NEON (ARMv8).
|
||||
*/
|
||||
namespace arm64 {
|
||||
} // namespace arm64
|
||||
} // namespace simdjson
|
||||
|
||||
#include "simdjson/arm64/implementation.h"
|
||||
|
||||
#include "simdjson/arm64/begin.h"
|
||||
|
||||
// Declarations
|
||||
#include "simdjson/generic/dom_parser_implementation.h"
|
||||
#include "simdjson/arm64/intrinsics.h"
|
||||
#include "simdjson/arm64/bitmanipulation.h"
|
||||
#include "simdjson/arm64/bitmask.h"
|
||||
#include "simdjson/arm64/simd.h"
|
||||
#include "simdjson/generic/jsoncharutils.h"
|
||||
#include "simdjson/generic/atomparsing.h"
|
||||
#include "simdjson/arm64/stringparsing.h"
|
||||
#include "simdjson/arm64/numberparsing.h"
|
||||
#include "simdjson/generic/amalgamated.h"
|
||||
#include "simdjson/arm64/end.h"
|
||||
|
||||
#endif // SIMDJSON_IMPLEMENTATION_ARM64
|
||||
|
||||
#endif // SIMDJSON_ARM64_H
|
||||
#endif // SIMDJSON_ARM64_H
|
||||
@@ -0,0 +1,33 @@
|
||||
#ifndef SIMDJSON_ARM64_BASE_H
|
||||
#define SIMDJSON_ARM64_BASE_H
|
||||
|
||||
#ifndef SIMDJSON_AMALGAMATED
|
||||
#include "simdjson/base.h"
|
||||
#endif // SIMDJSON_AMALGAMATED
|
||||
|
||||
namespace simdjson {
|
||||
/**
|
||||
* Implementation for NEON (ARMv8).
|
||||
*/
|
||||
namespace arm64 {
|
||||
|
||||
class implementation;
|
||||
|
||||
namespace {
|
||||
namespace simd {
|
||||
template <typename T> struct simd8;
|
||||
template <typename T> struct simd8x64;
|
||||
} // namespace simd
|
||||
} // unnamed namespace
|
||||
|
||||
} // namespace arm64
|
||||
} // namespace simdjson
|
||||
|
||||
#ifndef SIMDJSON_AMALGAMATED
|
||||
// If we're editing one of the files in this directory, begin the implementation!
|
||||
#ifndef SIMDJSON_IMPLEMENTATION
|
||||
#include "simdjson/arm64/begin.h"
|
||||
#endif
|
||||
#endif // SIMDJSON_AMALGAMATED
|
||||
|
||||
#endif // SIMDJSON_ARM64_BASE_H
|
||||
@@ -1 +1,8 @@
|
||||
#define SIMDJSON_IMPLEMENTATION arm64
|
||||
#include "simdjson/arm64/base.h"
|
||||
#include "simdjson/arm64/intrinsics.h"
|
||||
#include "simdjson/arm64/bitmanipulation.h"
|
||||
#include "simdjson/arm64/bitmask.h"
|
||||
#include "simdjson/arm64/numberparsing_defs.h"
|
||||
#include "simdjson/arm64/simd.h"
|
||||
#include "simdjson/arm64/stringparsing_defs.h"
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
#ifndef SIMDJSON_ARM64_BITMANIPULATION_H
|
||||
#define SIMDJSON_ARM64_BITMANIPULATION_H
|
||||
|
||||
#include "simdjson/arm64/base.h"
|
||||
#include "simdjson/arm64/intrinsics.h"
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
namespace {
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
#ifndef SIMDJSON_ARM64_BITMASK_H
|
||||
#define SIMDJSON_ARM64_BITMASK_H
|
||||
|
||||
#include "simdjson/arm64/base.h"
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
namespace {
|
||||
|
||||
@@ -1,17 +1,14 @@
|
||||
#ifndef SIMDJSON_ARM64_IMPLEMENTATION_H
|
||||
#define SIMDJSON_ARM64_IMPLEMENTATION_H
|
||||
|
||||
#include "simdjson/base.h"
|
||||
#include "simdjson/internal/isadetection.h"
|
||||
#ifndef SIMDJSON_AMALGAMATED
|
||||
#include "simdjson/implementation.h"
|
||||
#include "simdjson/internal/instruction_set.h"
|
||||
#endif // SIMDJSON_AMALGAMATED
|
||||
|
||||
namespace simdjson {
|
||||
namespace arm64 {
|
||||
|
||||
namespace {
|
||||
using namespace simdjson;
|
||||
using namespace simdjson::dom;
|
||||
}
|
||||
|
||||
/**
|
||||
* @private
|
||||
*/
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
// you use visual studio or other compilers.
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "simdjson/arm64/base.h"
|
||||
|
||||
static_assert(sizeof(uint8x16_t) <= simdjson::SIMDJSON_PADDING, "insufficient padding for arm64");
|
||||
|
||||
#endif // SIMDJSON_ARM64_INTRINSICS_H
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
#ifndef SIMDJSON_ARM64_NUMBERPARSING_H
|
||||
#define SIMDJSON_ARM64_NUMBERPARSING_H
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
namespace {
|
||||
|
||||
// we don't have SSE, so let us use a scalar function
|
||||
// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
|
||||
static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
|
||||
uint64_t val;
|
||||
std::memcpy(&val, chars, sizeof(uint64_t));
|
||||
val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8;
|
||||
val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16;
|
||||
return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32);
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
} // namespace simdjson
|
||||
|
||||
#define SIMDJSON_SWAR_NUMBER_PARSING 1
|
||||
|
||||
#include "simdjson/generic/numberparsing.h"
|
||||
|
||||
#endif // SIMDJSON_ARM64_NUMBERPARSING_H
|
||||
@@ -0,0 +1,57 @@
|
||||
#ifndef SIMDJSON_ARM64_NUMBERPARSING_DEFS_H
|
||||
#define SIMDJSON_ARM64_NUMBERPARSING_DEFS_H
|
||||
|
||||
#include "simdjson/arm64/base.h"
|
||||
#include "simdjson/arm64/intrinsics.h"
|
||||
|
||||
#if _M_ARM64
|
||||
// __umulh requires intrin.h
|
||||
#include <intrin.h>
|
||||
#endif // _M_ARM64
|
||||
|
||||
#include <cstring>
|
||||
|
||||
#ifndef SIMDJSON_AMALGAMATED
|
||||
#include "simdjson/internal/numberparsing_tables.h"
|
||||
#endif // SIMDJSON_AMALGAMATED
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
namespace numberparsing {
|
||||
|
||||
// we don't have SSE, so let us use a scalar function
|
||||
// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
|
||||
/** @private */
|
||||
static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
|
||||
uint64_t val;
|
||||
std::memcpy(&val, chars, sizeof(uint64_t));
|
||||
val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8;
|
||||
val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16;
|
||||
return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32);
|
||||
}
|
||||
|
||||
simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) {
|
||||
internal::value128 answer;
|
||||
#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS
|
||||
#ifdef _M_ARM64
|
||||
// ARM64 has native support for 64-bit multiplications, no need to emultate
|
||||
answer.high = __umulh(value1, value2);
|
||||
answer.low = value1 * value2;
|
||||
#else
|
||||
answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64
|
||||
#endif // _M_ARM64
|
||||
#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS
|
||||
__uint128_t r = (static_cast<__uint128_t>(value1)) * value2;
|
||||
answer.low = uint64_t(r);
|
||||
answer.high = uint64_t(r >> 64);
|
||||
#endif
|
||||
return answer;
|
||||
}
|
||||
|
||||
} // namespace numberparsing
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
} // namespace simdjson
|
||||
|
||||
#define SIMDJSON_SWAR_NUMBER_PARSING 1
|
||||
|
||||
#endif // SIMDJSON_ARM64_NUMBERPARSING_DEFS_H
|
||||
@@ -0,0 +1,8 @@
|
||||
#ifndef SIMDJSON_ARM64_ONDEMAND_H
|
||||
#define SIMDJSON_ARM64_ONDEMAND_H
|
||||
|
||||
#include "simdjson/arm64/begin.h"
|
||||
#include "simdjson/generic/ondemand/amalgamated.h"
|
||||
#include "simdjson/arm64/end.h"
|
||||
|
||||
#endif // SIMDJSON_ARM64_ONDEMAND_H
|
||||
@@ -1,11 +1,12 @@
|
||||
#ifndef SIMDJSON_ARM64_SIMD_H
|
||||
#define SIMDJSON_ARM64_SIMD_H
|
||||
|
||||
#include "simdjson/base.h"
|
||||
#include "simdjson/internal/simdprune_tables.h"
|
||||
#include "simdjson/arm64/base.h"
|
||||
#include "simdjson/arm64/bitmanipulation.h"
|
||||
#include <type_traits>
|
||||
|
||||
#ifndef SIMDJSON_AMALGAMATED
|
||||
#include "simdjson/internal/simdprune_tables.h"
|
||||
#endif // SIMDJSON_AMALGAMATED
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
#ifndef SIMDJSON_ARM64_SINGLESTAGE_H
|
||||
#define SIMDJSON_ARM64_SINGLESTAGE_H
|
||||
|
||||
#include "simdjson/arm64/begin.h"
|
||||
#include "simdjson/generic/singlestage/amalgamated.h"
|
||||
#include "simdjson/arm64/end.h"
|
||||
|
||||
#endif // SIMDJSON_ARM64_SINGLESTAGE_H
|
||||
+4
-4
@@ -1,7 +1,7 @@
|
||||
#ifndef SIMDJSON_ARM64_STRINGPARSING_H
|
||||
#define SIMDJSON_ARM64_STRINGPARSING_H
|
||||
#ifndef SIMDJSON_ARM64_STRINGPARSING_DEFS_H
|
||||
#define SIMDJSON_ARM64_STRINGPARSING_DEFS_H
|
||||
|
||||
#include "simdjson/base.h"
|
||||
#include "simdjson/arm64/base.h"
|
||||
#include "simdjson/arm64/simd.h"
|
||||
#include "simdjson/arm64/bitmanipulation.h"
|
||||
|
||||
@@ -48,4 +48,4 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_ARM64_STRINGPARSING_H
|
||||
#endif // SIMDJSON_ARM64_STRINGPARSING_DEFS_H
|
||||
+50
-16
@@ -1,26 +1,60 @@
|
||||
/**
|
||||
* @file Base declarations for all simdjson headers
|
||||
* @private
|
||||
*/
|
||||
#ifndef SIMDJSON_BASE_H
|
||||
#define SIMDJSON_BASE_H
|
||||
|
||||
#include "simdjson/compiler_check.h"
|
||||
#include "simdjson/common_defs.h"
|
||||
#include "simdjson/compiler_check.h"
|
||||
#include "simdjson/error.h"
|
||||
#include "simdjson/portability.h"
|
||||
|
||||
SIMDJSON_PUSH_DISABLE_WARNINGS
|
||||
SIMDJSON_DISABLE_UNDESIRED_WARNINGS
|
||||
/**
|
||||
* @brief The top level simdjson namespace, containing everything the library provides.
|
||||
*/
|
||||
namespace simdjson {
|
||||
|
||||
// Public API
|
||||
#include "simdjson/simdjson_version.h"
|
||||
#include "simdjson/error.h"
|
||||
#include "simdjson/minify.h"
|
||||
#include "simdjson/padded_string.h"
|
||||
#include "simdjson/padded_string_view.h"
|
||||
#include "simdjson/implementation.h"
|
||||
SIMDJSON_PUSH_DISABLE_UNUSED_WARNINGS
|
||||
|
||||
// Inline functions
|
||||
#include "simdjson/error-inl.h"
|
||||
#include "simdjson/padded_string-inl.h"
|
||||
#include "simdjson/padded_string_view-inl.h"
|
||||
/** The maximum document size supported by simdjson. */
|
||||
constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF;
|
||||
|
||||
SIMDJSON_POP_DISABLE_WARNINGS
|
||||
/**
|
||||
* The amount of padding needed in a buffer to parse JSON.
|
||||
*
|
||||
* The input buf should be readable up to buf + SIMDJSON_PADDING
|
||||
* this is a stopgap; there should be a better description of the
|
||||
* main loop and its behavior that abstracts over this
|
||||
* See https://github.com/simdjson/simdjson/issues/174
|
||||
*/
|
||||
constexpr size_t SIMDJSON_PADDING = 64;
|
||||
|
||||
#endif // SIMDJSON_BASE_H
|
||||
/**
|
||||
* By default, simdjson supports this many nested objects and arrays.
|
||||
*
|
||||
* This is the default for parser::max_depth().
|
||||
*/
|
||||
constexpr size_t DEFAULT_MAX_DEPTH = 1024;
|
||||
|
||||
SIMDJSON_POP_DISABLE_UNUSED_WARNINGS
|
||||
|
||||
class implementation;
|
||||
struct padded_string;
|
||||
class padded_string_view;
|
||||
enum class stage1_mode;
|
||||
|
||||
namespace internal {
|
||||
|
||||
template<typename T>
|
||||
class atomic_ptr;
|
||||
class dom_parser_implementation;
|
||||
class escape_json_string;
|
||||
class tape_ref;
|
||||
struct value128;
|
||||
enum class tape_type;
|
||||
|
||||
} // namespace internal
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_BASE_H
|
||||
|
||||
+20
-55
@@ -1,64 +1,29 @@
|
||||
#ifndef SIMDJSON_BUILTIN_H
|
||||
#define SIMDJSON_BUILTIN_H
|
||||
|
||||
#include "simdjson/implementations.h"
|
||||
#include "simdjson/builtin/base.h"
|
||||
#include "simdjson/builtin/implementation.h"
|
||||
|
||||
// Determine the best builtin implementation
|
||||
#ifndef SIMDJSON_BUILTIN_IMPLEMENTATION
|
||||
#if SIMDJSON_CAN_ALWAYS_RUN_ICELAKE
|
||||
#define SIMDJSON_BUILTIN_IMPLEMENTATION icelake
|
||||
#elif SIMDJSON_CAN_ALWAYS_RUN_HASWELL
|
||||
#define SIMDJSON_BUILTIN_IMPLEMENTATION haswell
|
||||
#elif SIMDJSON_CAN_ALWAYS_RUN_WESTMERE
|
||||
#define SIMDJSON_BUILTIN_IMPLEMENTATION westmere
|
||||
#elif SIMDJSON_CAN_ALWAYS_RUN_ARM64
|
||||
#define SIMDJSON_BUILTIN_IMPLEMENTATION arm64
|
||||
#elif SIMDJSON_CAN_ALWAYS_RUN_PPC64
|
||||
#define SIMDJSON_BUILTIN_IMPLEMENTATION ppc64
|
||||
#elif SIMDJSON_CAN_ALWAYS_RUN_FALLBACK
|
||||
#define SIMDJSON_BUILTIN_IMPLEMENTATION fallback
|
||||
#include "simdjson/generic/dependencies.h"
|
||||
|
||||
#define SIMDJSON_AMALGAMATED
|
||||
|
||||
#if SIMDJSON_BUILTIN_IMPLEMENTATION_IS(arm64)
|
||||
#include "simdjson/arm64.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(fallback)
|
||||
#include "simdjson/fallback.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(haswell)
|
||||
#include "simdjson/haswell.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(icelake)
|
||||
#include "simdjson/icelake.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(ppc64)
|
||||
#include "simdjson/ppc64.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(westmere)
|
||||
#include "simdjson/westmere.h"
|
||||
#else
|
||||
#error "All possible implementations (including fallback) have been disabled! simdjson will not run."
|
||||
#error Unknown SIMDJSON_BUILTIN_IMPLEMENTATION
|
||||
#endif
|
||||
#endif // SIMDJSON_BUILTIN_IMPLEMENTATION
|
||||
|
||||
#define SIMDJSON_IMPLEMENTATION SIMDJSON_BUILTIN_IMPLEMENTATION
|
||||
|
||||
// ondemand is only compiled as part of the builtin implementation at present
|
||||
|
||||
// Interface declarations
|
||||
#include "simdjson/generic/implementation_simdjson_result_base.h"
|
||||
#include "simdjson/generic/ondemand.h"
|
||||
|
||||
// Inline definitions
|
||||
#include "simdjson/generic/implementation_simdjson_result_base-inl.h"
|
||||
#include "simdjson/generic/ondemand-inl.h"
|
||||
|
||||
#undef SIMDJSON_IMPLEMENTATION
|
||||
|
||||
namespace simdjson {
|
||||
/**
|
||||
* Represents the best statically linked simdjson implementation that can be used by the compiling
|
||||
* program.
|
||||
*
|
||||
* Detects what options the program is compiled against, and picks the minimum implementation that
|
||||
* will work on any computer that can run the program. For example, if you compile with g++
|
||||
* -march=westmere, it will pick the westmere implementation. The haswell implementation will
|
||||
* still be available, and can be selected at runtime, but the builtin implementation (and any
|
||||
* code that uses it) will use westmere.
|
||||
*/
|
||||
namespace builtin = SIMDJSON_BUILTIN_IMPLEMENTATION;
|
||||
/**
|
||||
* @copydoc simdjson::SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand
|
||||
*/
|
||||
namespace ondemand = SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand;
|
||||
/**
|
||||
* Function which returns a pointer to an implementation matching the "builtin" implementation.
|
||||
* The builtin implementation is the best statically linked simdjson implementation that can be used by the compiling
|
||||
* program. If you compile with g++ -march=haswell, this will return the haswell implementation.
|
||||
* It is handy to be able to check what builtin was used: builtin_implementation()->name().
|
||||
*/
|
||||
const implementation * builtin_implementation();
|
||||
} // namespace simdjson
|
||||
#undef SIMDJSON_AMALGAMATED
|
||||
|
||||
#endif // SIMDJSON_BUILTIN_H
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
#ifndef SIMDJSON_BUILTIN_BASE_H
|
||||
#define SIMDJSON_BUILTIN_BASE_H
|
||||
|
||||
#include "simdjson/base.h"
|
||||
#include "simdjson/implementation_detection.h"
|
||||
|
||||
namespace simdjson {
|
||||
#if SIMDJSON_BUILTIN_IMPLEMENTATION_IS(arm64)
|
||||
namespace arm64 {}
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(fallback)
|
||||
namespace fallback {}
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(haswell)
|
||||
namespace haswell {}
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(icelake)
|
||||
namespace icelake {}
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(ppc64)
|
||||
namespace ppc64 {}
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(westmere)
|
||||
namespace westmere {}
|
||||
#else
|
||||
#error Unknown SIMDJSON_BUILTIN_IMPLEMENTATION
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Represents the best statically linked simdjson implementation that can be used by the compiling
|
||||
* program.
|
||||
*
|
||||
* Detects what options the program is compiled against, and picks the minimum implementation that
|
||||
* will work on any computer that can run the program. For example, if you compile with g++
|
||||
* -march=westmere, it will pick the westmere implementation. The haswell implementation will
|
||||
* still be available, and can be selected at runtime, but the builtin implementation (and any
|
||||
* code that uses it) will use westmere.
|
||||
*/
|
||||
namespace builtin = SIMDJSON_BUILTIN_IMPLEMENTATION;
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_BUILTIN_BASE_H
|
||||
@@ -0,0 +1,38 @@
|
||||
#ifndef SIMDJSON_BUILTIN_IMPLEMENTATION_H
|
||||
#define SIMDJSON_BUILTIN_IMPLEMENTATION_H
|
||||
|
||||
#include "simdjson/builtin/base.h"
|
||||
|
||||
#include "simdjson/generic/dependencies.h"
|
||||
|
||||
#define SIMDJSON_AMALGAMATED
|
||||
|
||||
#if SIMDJSON_BUILTIN_IMPLEMENTATION_IS(arm64)
|
||||
#include "simdjson/arm64/implementation.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(fallback)
|
||||
#include "simdjson/fallback/implementation.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(haswell)
|
||||
#include "simdjson/haswell/implementation.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(icelake)
|
||||
#include "simdjson/icelake/implementation.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(ppc64)
|
||||
#include "simdjson/ppc64/implementation.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(westmere)
|
||||
#include "simdjson/westmere/implementation.h"
|
||||
#else
|
||||
#error Unknown SIMDJSON_BUILTIN_IMPLEMENTATION
|
||||
#endif
|
||||
|
||||
#undef SIMDJSON_AMALGAMATED
|
||||
|
||||
namespace simdjson {
|
||||
/**
|
||||
* Function which returns a pointer to an implementation matching the "builtin" implementation.
|
||||
* The builtin implementation is the best statically linked simdjson implementation that can be used by the compiling
|
||||
* program. If you compile with g++ -march=haswell, this will return the haswell implementation.
|
||||
* It is handy to be able to check what builtin was used: builtin_implementation()->name().
|
||||
*/
|
||||
const implementation * builtin_implementation();
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_BUILTIN_IMPLEMENTATION_H
|
||||
@@ -0,0 +1,36 @@
|
||||
#ifndef SIMDJSON_BUILTIN_ONDEMAND_H
|
||||
#define SIMDJSON_BUILTIN_ONDEMAND_H
|
||||
|
||||
#include "simdjson/builtin.h"
|
||||
#include "simdjson/builtin/base.h"
|
||||
|
||||
#include "simdjson/generic/ondemand/dependencies.h"
|
||||
|
||||
#define SIMDJSON_AMALGAMATED
|
||||
|
||||
#if SIMDJSON_BUILTIN_IMPLEMENTATION_IS(arm64)
|
||||
#include "simdjson/arm64/ondemand.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(fallback)
|
||||
#include "simdjson/fallback/ondemand.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(haswell)
|
||||
#include "simdjson/haswell/ondemand.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(icelake)
|
||||
#include "simdjson/icelake/ondemand.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(ppc64)
|
||||
#include "simdjson/ppc64/ondemand.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(westmere)
|
||||
#include "simdjson/westmere/ondemand.h"
|
||||
#else
|
||||
#error Unknown SIMDJSON_BUILTIN_IMPLEMENTATION
|
||||
#endif
|
||||
|
||||
#undef SIMDJSON_AMALGAMATED
|
||||
|
||||
namespace simdjson {
|
||||
/**
|
||||
* @copydoc simdjson::SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand
|
||||
*/
|
||||
namespace ondemand = SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand;
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_BUILTIN_ONDEMAND_H
|
||||
@@ -0,0 +1,36 @@
|
||||
#ifndef SIMDJSON_BUILTIN_SINGLESTAGE_H
|
||||
#define SIMDJSON_BUILTIN_SINGLESTAGE_H
|
||||
|
||||
#include "simdjson/builtin.h"
|
||||
#include "simdjson/builtin/base.h"
|
||||
|
||||
#include "simdjson/generic/singlestage/dependencies.h"
|
||||
|
||||
#define SIMDJSON_AMALGAMATED
|
||||
|
||||
#if SIMDJSON_BUILTIN_IMPLEMENTATION_IS(arm64)
|
||||
#include "simdjson/arm64/singlestage.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(fallback)
|
||||
#include "simdjson/fallback/singlestage.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(haswell)
|
||||
#include "simdjson/haswell/singlestage.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(icelake)
|
||||
#include "simdjson/icelake/singlestage.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(ppc64)
|
||||
#include "simdjson/ppc64/singlestage.h"
|
||||
#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(westmere)
|
||||
#include "simdjson/westmere/singlestage.h"
|
||||
#else
|
||||
#error Unknown SIMDJSON_BUILTIN_IMPLEMENTATION
|
||||
#endif
|
||||
|
||||
#undef SIMDJSON_AMALGAMATED
|
||||
|
||||
namespace simdjson {
|
||||
/**
|
||||
* @copydoc simdjson::SIMDJSON_BUILTIN_IMPLEMENTATION::singlestage
|
||||
*/
|
||||
namespace singlestage = SIMDJSON_BUILTIN_IMPLEMENTATION::singlestage;
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_BUILTIN_SINGLESTAGE_H
|
||||
@@ -2,10 +2,10 @@
|
||||
#define SIMDJSON_COMMON_DEFS_H
|
||||
|
||||
#include <cassert>
|
||||
#include "simdjson/compiler_check.h"
|
||||
#include "simdjson/portability.h"
|
||||
|
||||
namespace simdjson {
|
||||
|
||||
namespace internal {
|
||||
/**
|
||||
* @private
|
||||
@@ -20,7 +20,6 @@ char *to_chars(char *first, const char *last, double value);
|
||||
*/
|
||||
double from_chars(const char *first) noexcept;
|
||||
double from_chars(const char *first, const char* end) noexcept;
|
||||
|
||||
}
|
||||
|
||||
#ifndef SIMDJSON_EXCEPTIONS
|
||||
@@ -31,26 +30,6 @@ double from_chars(const char *first, const char* end) noexcept;
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/** The maximum document size supported by simdjson. */
|
||||
constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF;
|
||||
|
||||
/**
|
||||
* The amount of padding needed in a buffer to parse JSON.
|
||||
*
|
||||
* The input buf should be readable up to buf + SIMDJSON_PADDING
|
||||
* this is a stopgap; there should be a better description of the
|
||||
* main loop and its behavior that abstracts over this
|
||||
* See https://github.com/simdjson/simdjson/issues/174
|
||||
*/
|
||||
constexpr size_t SIMDJSON_PADDING = 64;
|
||||
|
||||
/**
|
||||
* By default, simdjson supports this many nested objects and arrays.
|
||||
*
|
||||
* This is the default for parser::max_depth().
|
||||
*/
|
||||
constexpr size_t DEFAULT_MAX_DEPTH = 1024;
|
||||
|
||||
} // namespace simdjson
|
||||
|
||||
#if defined(__GNUC__)
|
||||
@@ -105,6 +84,9 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
|
||||
#define SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING
|
||||
#define SIMDJSON_POP_DISABLE_WARNINGS __pragma(warning( pop ))
|
||||
|
||||
#define SIMDJSON_PUSH_DISABLE_UNUSED_WARNINGS
|
||||
#define SIMDJSON_POP_DISABLE_UNUSED_WARNINGS
|
||||
|
||||
#else // SIMDJSON_REGULAR_VISUAL_STUDIO
|
||||
|
||||
#define simdjson_really_inline inline __attribute__((always_inline))
|
||||
@@ -150,7 +132,8 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
|
||||
SIMDJSON_DISABLE_GCC_WARNING(-Wshadow) \
|
||||
SIMDJSON_DISABLE_GCC_WARNING(-Wunused-parameter) \
|
||||
SIMDJSON_DISABLE_GCC_WARNING(-Wunused-variable) \
|
||||
SIMDJSON_DISABLE_GCC_WARNING(-Wmaybe-uninitialized)
|
||||
SIMDJSON_DISABLE_GCC_WARNING(-Wmaybe-uninitialized) \
|
||||
SIMDJSON_DISABLE_GCC_WARNING(-Wformat-security)
|
||||
#endif // __clang__
|
||||
|
||||
#define SIMDJSON_PRAGMA(P) _Pragma(#P)
|
||||
@@ -164,6 +147,10 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
|
||||
#define SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING SIMDJSON_DISABLE_GCC_WARNING(-Wstrict-overflow)
|
||||
#define SIMDJSON_POP_DISABLE_WARNINGS _Pragma("GCC diagnostic pop")
|
||||
|
||||
#define SIMDJSON_PUSH_DISABLE_UNUSED_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \
|
||||
SIMDJSON_DISABLE_GCC_WARNING(-Wunused)
|
||||
#define SIMDJSON_POP_DISABLE_UNUSED_WARNINGS SIMDJSON_POP_DISABLE_WARNINGS
|
||||
|
||||
|
||||
|
||||
#endif // MSC_VER
|
||||
@@ -322,7 +309,6 @@ namespace std {
|
||||
# define simdjson_fallthrough do {} while (0) /* fallthrough */
|
||||
#endif // simdjson_fallthrough
|
||||
|
||||
|
||||
#if SIMDJSON_DEVELOPMENT_CHECKS
|
||||
#define SIMDJSON_DEVELOPMENT_ASSERT(expr) do { assert ((expr)); } while (0)
|
||||
#else
|
||||
@@ -333,4 +319,25 @@ namespace std {
|
||||
#define SIMDJSON_UTF8VALIDATION 1
|
||||
#endif
|
||||
|
||||
#ifdef __has_include
|
||||
// How do we detect that a compiler supports vbmi2?
|
||||
// For sure if the following header is found, we are ok?
|
||||
#if __has_include(<avx512vbmi2intrin.h>)
|
||||
#define SIMDJSON_COMPILER_SUPPORTS_VBMI2 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#if _MSC_VER >= 1920
|
||||
// Visual Studio 2019 and up support VBMI2 under x64 even if the header
|
||||
// avx512vbmi2intrin.h is not found.
|
||||
#define SIMDJSON_COMPILER_SUPPORTS_VBMI2 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// By default, we allow AVX512.
|
||||
#ifndef SIMDJSON_AVX512_ALLOWED
|
||||
#define SIMDJSON_AVX512_ALLOWED 1
|
||||
#endif
|
||||
|
||||
#endif // SIMDJSON_COMMON_DEFS_H
|
||||
|
||||
@@ -1,11 +1,7 @@
|
||||
#ifndef SIMDJSON_DOM_H
|
||||
#define SIMDJSON_DOM_H
|
||||
|
||||
#include "simdjson/base.h"
|
||||
|
||||
SIMDJSON_PUSH_DISABLE_WARNINGS
|
||||
SIMDJSON_DISABLE_UNDESIRED_WARNINGS
|
||||
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/dom/array.h"
|
||||
#include "simdjson/dom/document_stream.h"
|
||||
#include "simdjson/dom/document.h"
|
||||
@@ -30,6 +26,4 @@ SIMDJSON_DISABLE_UNDESIRED_WARNINGS
|
||||
#include "simdjson/internal/tape_ref-inl.h"
|
||||
#include "simdjson/dom/serialization-inl.h"
|
||||
|
||||
SIMDJSON_POP_DISABLE_WARNINGS
|
||||
|
||||
#endif // SIMDJSON_DOM_H
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
#ifndef SIMDJSON_INLINE_ARRAY_H
|
||||
#define SIMDJSON_INLINE_ARRAY_H
|
||||
#ifndef SIMDJSON_ARRAY_INL_H
|
||||
#define SIMDJSON_ARRAY_INL_H
|
||||
|
||||
// Inline implementations go in here.
|
||||
#include <utility>
|
||||
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/dom/array.h"
|
||||
#include "simdjson/dom/element.h"
|
||||
#include <utility>
|
||||
#include "simdjson/error-inl.h"
|
||||
#include "simdjson/internal/tape_ref-inl.h"
|
||||
|
||||
#include <limits>
|
||||
|
||||
namespace simdjson {
|
||||
|
||||
@@ -170,4 +174,4 @@ static_assert(std::ranges::sized_range<simdjson::simdjson_result<simdjson::dom::
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
#endif // defined(__cpp_lib_ranges)
|
||||
|
||||
#endif // SIMDJSON_INLINE_ARRAY_H
|
||||
#endif // SIMDJSON_ARRAY_INL_H
|
||||
|
||||
@@ -1,21 +1,12 @@
|
||||
#ifndef SIMDJSON_DOM_ARRAY_H
|
||||
#define SIMDJSON_DOM_ARRAY_H
|
||||
|
||||
#include "simdjson/common_defs.h"
|
||||
#include "simdjson/error.h"
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/internal/tape_ref.h"
|
||||
|
||||
namespace simdjson {
|
||||
|
||||
namespace internal {
|
||||
template<typename T>
|
||||
class string_builder;
|
||||
}
|
||||
namespace dom {
|
||||
|
||||
class document;
|
||||
class element;
|
||||
|
||||
/**
|
||||
* JSON array.
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
#ifndef SIMDJSON_DOM_BASE_H
|
||||
#define SIMDJSON_DOM_BASE_H
|
||||
|
||||
#include "simdjson/base.h"
|
||||
|
||||
namespace simdjson {
|
||||
|
||||
/**
|
||||
* @brief A DOM API on top of the simdjson parser.
|
||||
*/
|
||||
namespace dom {
|
||||
|
||||
/** The default batch size for parser.parse_many() and parser.load_many() */
|
||||
static constexpr size_t DEFAULT_BATCH_SIZE = 1000000;
|
||||
/**
|
||||
* Some adversary might try to set the batch size to 0 or 1, which might cause problems.
|
||||
* We set a minimum of 32B since anything else is highly likely to be an error. In practice,
|
||||
* most users will want a much larger batch size.
|
||||
*
|
||||
* All non-negative MINIMAL_BATCH_SIZE values should be 'safe' except that, obviously, no JSON
|
||||
* document can ever span 0 or 1 byte and that very large values would create memory allocation issues.
|
||||
*/
|
||||
static constexpr size_t MINIMAL_BATCH_SIZE = 32;
|
||||
|
||||
/**
|
||||
* It is wasteful to allocate memory for tiny documents (e.g., 4 bytes).
|
||||
*/
|
||||
static constexpr size_t MINIMAL_DOCUMENT_CAPACITY = 32;
|
||||
|
||||
class array;
|
||||
class document;
|
||||
class document_stream;
|
||||
class element;
|
||||
class key_value_pair;
|
||||
class object;
|
||||
class parser;
|
||||
|
||||
#ifdef SIMDJSON_THREADS_ENABLED
|
||||
struct stage1_worker;
|
||||
#endif // SIMDJSON_THREADS_ENABLED
|
||||
|
||||
} // namespace dom
|
||||
|
||||
namespace internal {
|
||||
|
||||
template<typename T>
|
||||
class string_builder;
|
||||
class tape_ref;
|
||||
|
||||
} // namespace internal
|
||||
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_DOM_BASE_H
|
||||
@@ -1,14 +1,13 @@
|
||||
#ifndef SIMDJSON_INLINE_DOCUMENT_H
|
||||
#define SIMDJSON_INLINE_DOCUMENT_H
|
||||
#ifndef SIMDJSON_DOCUMENT_INL_H
|
||||
#define SIMDJSON_DOCUMENT_INL_H
|
||||
|
||||
// Inline implementations go in here.
|
||||
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/dom/document.h"
|
||||
#include "simdjson/dom/element.h"
|
||||
#include "simdjson/internal/tape_ref.h"
|
||||
#include "simdjson/internal/tape_ref-inl.h"
|
||||
#include "simdjson/internal/jsonformatutils.h"
|
||||
#include <ostream>
|
||||
#include <cstring>
|
||||
|
||||
namespace simdjson {
|
||||
namespace dom {
|
||||
@@ -155,4 +154,4 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept {
|
||||
} // namespace dom
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_INLINE_DOCUMENT_H
|
||||
#endif // SIMDJSON_DOCUMENT_INL_H
|
||||
|
||||
@@ -1,15 +1,13 @@
|
||||
#ifndef SIMDJSON_DOM_DOCUMENT_H
|
||||
#define SIMDJSON_DOM_DOCUMENT_H
|
||||
|
||||
#include "simdjson/common_defs.h"
|
||||
#include "simdjson/dom/base.h"
|
||||
|
||||
#include <memory>
|
||||
#include <ostream>
|
||||
|
||||
namespace simdjson {
|
||||
namespace dom {
|
||||
|
||||
class element;
|
||||
|
||||
/**
|
||||
* A parsed JSON document.
|
||||
*
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
#ifndef SIMDJSON_INLINE_DOCUMENT_STREAM_H
|
||||
#define SIMDJSON_INLINE_DOCUMENT_STREAM_H
|
||||
#ifndef SIMDJSON_DOCUMENT_STREAM_INL_H
|
||||
#define SIMDJSON_DOCUMENT_STREAM_INL_H
|
||||
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/dom/document_stream.h"
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <stdexcept>
|
||||
#include "simdjson/dom/element-inl.h"
|
||||
#include "simdjson/dom/parser-inl.h"
|
||||
#include "simdjson/error-inl.h"
|
||||
#include "simdjson/internal/dom_parser_implementation.h"
|
||||
|
||||
namespace simdjson {
|
||||
namespace dom {
|
||||
|
||||
#ifdef SIMDJSON_THREADS_ENABLED
|
||||
|
||||
inline void stage1_worker::finish() {
|
||||
// After calling "run" someone would call finish() to wait
|
||||
// for the end of the processing.
|
||||
@@ -337,4 +341,4 @@ simdjson_inline dom::document_stream::iterator simdjson_result<dom::document_str
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
|
||||
} // namespace simdjson
|
||||
#endif // SIMDJSON_INLINE_DOCUMENT_STREAM_H
|
||||
#endif // SIMDJSON_DOCUMENT_STREAM_INL_H
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
#ifndef SIMDJSON_DOCUMENT_STREAM_H
|
||||
#define SIMDJSON_DOCUMENT_STREAM_H
|
||||
|
||||
#include "simdjson/common_defs.h"
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/dom/parser.h"
|
||||
#include "simdjson/error.h"
|
||||
|
||||
#ifdef SIMDJSON_THREADS_ENABLED
|
||||
#include <thread>
|
||||
#include <mutex>
|
||||
@@ -13,7 +13,6 @@
|
||||
namespace simdjson {
|
||||
namespace dom {
|
||||
|
||||
|
||||
#ifdef SIMDJSON_THREADS_ENABLED
|
||||
/** @private Custom worker class **/
|
||||
struct stage1_worker {
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
#ifndef SIMDJSON_INLINE_ELEMENT_H
|
||||
#define SIMDJSON_INLINE_ELEMENT_H
|
||||
#ifndef SIMDJSON_ELEMENT_INL_H
|
||||
#define SIMDJSON_ELEMENT_INL_H
|
||||
|
||||
#include "simdjson/dom/array.h"
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/dom/element.h"
|
||||
#include "simdjson/dom/document.h"
|
||||
#include "simdjson/dom/object.h"
|
||||
#include <cstring>
|
||||
#include <utility>
|
||||
#include "simdjson/internal/tape_type.h"
|
||||
|
||||
#include <ostream>
|
||||
#include <limits>
|
||||
namespace simdjson {
|
||||
|
||||
//
|
||||
@@ -438,4 +440,4 @@ inline std::ostream& operator<<(std::ostream& out, element_type type) {
|
||||
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_INLINE_ELEMENT_H
|
||||
#endif // SIMDJSON_ELEMENT_INL_H
|
||||
|
||||
@@ -1,20 +1,11 @@
|
||||
#ifndef SIMDJSON_DOM_ELEMENT_H
|
||||
#define SIMDJSON_DOM_ELEMENT_H
|
||||
|
||||
#include "simdjson/common_defs.h"
|
||||
#include "simdjson/error.h"
|
||||
#include "simdjson/internal/tape_ref.h"
|
||||
#include <ostream>
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/dom/array.h"
|
||||
|
||||
namespace simdjson {
|
||||
namespace internal {
|
||||
template<typename T>
|
||||
class string_builder;
|
||||
}
|
||||
namespace dom {
|
||||
class array;
|
||||
class document;
|
||||
class object;
|
||||
|
||||
/**
|
||||
* The actual concrete type of a JSON element
|
||||
@@ -537,7 +528,6 @@ public:
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
};
|
||||
|
||||
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_DOM_DOCUMENT_H
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
#ifndef SIMDJSON_DOM_JSONPARSER_H
|
||||
#define SIMDJSON_DOM_JSONPARSER_H
|
||||
|
||||
#include "simdjson/dom/document.h"
|
||||
#include "simdjson/dom/parsedjson.h"
|
||||
#include "simdjson/jsonioutil.h"
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/dom/parser.h"
|
||||
#include "simdjson/dom/element.h"
|
||||
|
||||
namespace simdjson {
|
||||
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
#ifndef SIMDJSON_INLINE_OBJECT_H
|
||||
#define SIMDJSON_INLINE_OBJECT_H
|
||||
#ifndef SIMDJSON_OBJECT_INL_H
|
||||
#define SIMDJSON_OBJECT_INL_H
|
||||
|
||||
#include "simdjson/dom/element.h"
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/dom/object.h"
|
||||
#include "simdjson/portability.h"
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include "simdjson/dom/document.h"
|
||||
|
||||
namespace simdjson {
|
||||
|
||||
@@ -253,4 +251,4 @@ static_assert(std::ranges::sized_range<simdjson::simdjson_result<simdjson::dom::
|
||||
#endif // SIMDJSON_EXCEPTIONS
|
||||
#endif // defined(__cpp_lib_ranges)
|
||||
|
||||
#endif // SIMDJSON_INLINE_OBJECT_H
|
||||
#endif // SIMDJSON_OBJECT_INL_H
|
||||
|
||||
@@ -1,21 +1,13 @@
|
||||
#ifndef SIMDJSON_DOM_OBJECT_H
|
||||
#define SIMDJSON_DOM_OBJECT_H
|
||||
|
||||
#include "simdjson/common_defs.h"
|
||||
#include "simdjson/error.h"
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/dom/element.h"
|
||||
#include "simdjson/internal/tape_ref.h"
|
||||
|
||||
namespace simdjson {
|
||||
namespace internal {
|
||||
template<typename T>
|
||||
class string_builder;
|
||||
}
|
||||
namespace dom {
|
||||
|
||||
class document;
|
||||
class element;
|
||||
class key_value_pair;
|
||||
|
||||
/**
|
||||
* JSON object.
|
||||
*/
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
#ifndef SIMDJSON_DOM_PARSEDJSON_H
|
||||
#define SIMDJSON_DOM_PARSEDJSON_H
|
||||
|
||||
#include "simdjson/dom/document.h"
|
||||
#include "simdjson/dom/base.h"
|
||||
|
||||
namespace simdjson {
|
||||
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
#ifndef SIMDJSON_INLINE_PARSEDJSON_ITERATOR_H
|
||||
#define SIMDJSON_INLINE_PARSEDJSON_ITERATOR_H
|
||||
#ifndef SIMDJSON_PARSEDJSON_ITERATOR_INL_H
|
||||
#define SIMDJSON_PARSEDJSON_ITERATOR_INL_H
|
||||
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/dom/parsedjson_iterator.h"
|
||||
#include "simdjson/portability.h"
|
||||
#include <cstring>
|
||||
#include "simdjson/internal/tape_ref-inl.h"
|
||||
#include "simdjson/internal/jsonformatutils.h"
|
||||
#include <ostream>
|
||||
#include <iterator>
|
||||
|
||||
#ifndef SIMDJSON_DISABLE_DEPRECATED_API
|
||||
|
||||
@@ -197,6 +200,12 @@ void dom::parser::Iterator::to_start_scope() {
|
||||
current_type = uint8_t(current_val >> 56);
|
||||
}
|
||||
|
||||
inline void dom::parser::Iterator::rewind() {
|
||||
while (up())
|
||||
;
|
||||
}
|
||||
|
||||
|
||||
bool dom::parser::Iterator::next() {
|
||||
size_t npos;
|
||||
if ((current_type == '[') || (current_type == '{')) {
|
||||
@@ -365,6 +374,48 @@ bool dom::parser::Iterator::move_to(const char *pointer,
|
||||
return found;
|
||||
}
|
||||
|
||||
inline bool dom::parser::Iterator::move_to(const std::string &pointer) {
|
||||
return move_to(pointer.c_str(), uint32_t(pointer.length()));
|
||||
}
|
||||
|
||||
inline int64_t dom::parser::Iterator::get_integer() const {
|
||||
if (location + 1 >= tape_length) {
|
||||
return 0; // default value in case of error
|
||||
}
|
||||
return static_cast<int64_t>(doc.tape[location + 1]);
|
||||
}
|
||||
|
||||
inline uint64_t dom::parser::Iterator::get_unsigned_integer() const {
|
||||
if (location + 1 >= tape_length) {
|
||||
return 0; // default value in case of error
|
||||
}
|
||||
return doc.tape[location + 1];
|
||||
}
|
||||
|
||||
inline const char * dom::parser::Iterator::get_string() const {
|
||||
return reinterpret_cast<const char *>(
|
||||
doc.string_buf.get() + (current_val & internal::JSON_VALUE_MASK) + sizeof(uint32_t));
|
||||
}
|
||||
|
||||
inline uint32_t dom::parser::Iterator::get_string_length() const {
|
||||
uint32_t answer;
|
||||
std::memcpy(&answer,
|
||||
reinterpret_cast<const char *>(doc.string_buf.get() +
|
||||
(current_val & internal::JSON_VALUE_MASK)),
|
||||
sizeof(uint32_t));
|
||||
return answer;
|
||||
}
|
||||
|
||||
inline double dom::parser::Iterator::get_double() const {
|
||||
if (location + 1 >= tape_length) {
|
||||
return std::numeric_limits<double>::quiet_NaN(); // default value in
|
||||
// case of error
|
||||
}
|
||||
double answer;
|
||||
std::memcpy(&answer, &doc.tape[location + 1], sizeof(answer));
|
||||
return answer;
|
||||
}
|
||||
|
||||
bool dom::parser::Iterator::relative_move_to(const char *pointer,
|
||||
uint32_t length) {
|
||||
if (length == 0) {
|
||||
@@ -486,4 +537,4 @@ SIMDJSON_POP_DISABLE_WARNINGS
|
||||
#endif // SIMDJSON_DISABLE_DEPRECATED_API
|
||||
|
||||
|
||||
#endif // SIMDJSON_INLINE_PARSEDJSON_ITERATOR_H
|
||||
#endif // SIMDJSON_PARSEDJSON_ITERATOR_INL_H
|
||||
|
||||
@@ -3,16 +3,8 @@
|
||||
#ifndef SIMDJSON_DOM_PARSEDJSON_ITERATOR_H
|
||||
#define SIMDJSON_DOM_PARSEDJSON_ITERATOR_H
|
||||
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <ostream>
|
||||
#include <iterator>
|
||||
#include <limits>
|
||||
#include <stdexcept>
|
||||
|
||||
#include "simdjson/dom/document.h"
|
||||
#include "simdjson/dom/parsedjson.h"
|
||||
#include "simdjson/internal/jsonformatutils.h"
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/dom/parser.h"
|
||||
|
||||
#ifndef SIMDJSON_DISABLE_DEPRECATED_API
|
||||
|
||||
@@ -52,51 +44,23 @@ public:
|
||||
}
|
||||
|
||||
// get the int64_t value at this node; valid only if get_type is "l"
|
||||
inline int64_t get_integer() const {
|
||||
if (location + 1 >= tape_length) {
|
||||
return 0; // default value in case of error
|
||||
}
|
||||
return static_cast<int64_t>(doc.tape[location + 1]);
|
||||
}
|
||||
inline int64_t get_integer() const;
|
||||
|
||||
// get the value as uint64; valid only if if get_type is "u"
|
||||
inline uint64_t get_unsigned_integer() const {
|
||||
if (location + 1 >= tape_length) {
|
||||
return 0; // default value in case of error
|
||||
}
|
||||
return doc.tape[location + 1];
|
||||
}
|
||||
inline uint64_t get_unsigned_integer() const;
|
||||
|
||||
// get the string value at this node (NULL ended); valid only if get_type is "
|
||||
// note that tabs, and line endings are escaped in the returned value (see
|
||||
// print_with_escapes) return value is valid UTF-8, it may contain NULL chars
|
||||
// within the string: get_string_length determines the true string length.
|
||||
inline const char *get_string() const {
|
||||
return reinterpret_cast<const char *>(
|
||||
doc.string_buf.get() + (current_val & internal::JSON_VALUE_MASK) + sizeof(uint32_t));
|
||||
}
|
||||
inline const char *get_string() const;
|
||||
|
||||
// return the length of the string in bytes
|
||||
inline uint32_t get_string_length() const {
|
||||
uint32_t answer;
|
||||
std::memcpy(&answer,
|
||||
reinterpret_cast<const char *>(doc.string_buf.get() +
|
||||
(current_val & internal::JSON_VALUE_MASK)),
|
||||
sizeof(uint32_t));
|
||||
return answer;
|
||||
}
|
||||
inline uint32_t get_string_length() const;
|
||||
|
||||
// get the double value at this node; valid only if
|
||||
// get_type() is "d"
|
||||
inline double get_double() const {
|
||||
if (location + 1 >= tape_length) {
|
||||
return std::numeric_limits<double>::quiet_NaN(); // default value in
|
||||
// case of error
|
||||
}
|
||||
double answer;
|
||||
std::memcpy(&answer, &doc.tape[location + 1], sizeof(answer));
|
||||
return answer;
|
||||
}
|
||||
inline double get_double() const;
|
||||
|
||||
inline bool is_object_or_array() const { return is_object() || is_array(); }
|
||||
|
||||
@@ -190,9 +154,7 @@ public:
|
||||
// "If a referenced member name is not unique in an object, the member that
|
||||
// is referenced is undefined, and evaluation fails". Here we just return
|
||||
// the first corresponding value.
|
||||
inline bool move_to(const std::string &pointer) {
|
||||
return move_to(pointer.c_str(), uint32_t(pointer.length()));
|
||||
}
|
||||
inline bool move_to(const std::string &pointer);
|
||||
|
||||
private:
|
||||
// Almost the same as move_to(), except it searches from the current
|
||||
@@ -240,10 +202,7 @@ public:
|
||||
// a scope is a series of nodes at the same level
|
||||
inline void to_start_scope();
|
||||
|
||||
inline void rewind() {
|
||||
while (up())
|
||||
;
|
||||
}
|
||||
inline void rewind();
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
#ifndef SIMDJSON_INLINE_PARSER_H
|
||||
#define SIMDJSON_INLINE_PARSER_H
|
||||
#ifndef SIMDJSON_PARSER_INL_H
|
||||
#define SIMDJSON_PARSER_INL_H
|
||||
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/dom/document_stream.h"
|
||||
#include "simdjson/dom/parser.h"
|
||||
#include "simdjson/implementation.h"
|
||||
#include "simdjson/internal/jsonformatutils.h"
|
||||
#include "simdjson/portability.h"
|
||||
#include <cstdio>
|
||||
#include "simdjson/internal/dom_parser_implementation.h"
|
||||
|
||||
#include "simdjson/error-inl.h"
|
||||
#include "simdjson/padded_string-inl.h"
|
||||
#include "simdjson/dom/element-inl.h"
|
||||
|
||||
#include <climits>
|
||||
|
||||
namespace simdjson {
|
||||
@@ -236,4 +239,4 @@ simdjson_inline void parser::set_max_capacity(size_t max_capacity) noexcept {
|
||||
} // namespace dom
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_INLINE_PARSER_H
|
||||
#endif // SIMDJSON_PARSER_INL_H
|
||||
|
||||
@@ -1,41 +1,13 @@
|
||||
#ifndef SIMDJSON_DOM_PARSER_H
|
||||
#define SIMDJSON_DOM_PARSER_H
|
||||
|
||||
#include "simdjson/common_defs.h"
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/dom/document.h"
|
||||
#include "simdjson/error.h"
|
||||
#include "simdjson/internal/dom_parser_implementation.h"
|
||||
#include "simdjson/internal/tape_ref.h"
|
||||
#include "simdjson/padded_string.h"
|
||||
#include "simdjson/portability.h"
|
||||
#include <memory>
|
||||
#include <ostream>
|
||||
#include <string>
|
||||
|
||||
namespace simdjson {
|
||||
|
||||
namespace dom {
|
||||
|
||||
class document_stream;
|
||||
class element;
|
||||
|
||||
/** The default batch size for parser.parse_many() and parser.load_many() */
|
||||
static constexpr size_t DEFAULT_BATCH_SIZE = 1000000;
|
||||
/**
|
||||
* Some adversary might try to set the batch size to 0 or 1, which might cause problems.
|
||||
* We set a minimum of 32B since anything else is highly likely to be an error. In practice,
|
||||
* most users will want a much larger batch size.
|
||||
*
|
||||
* All non-negative MINIMAL_BATCH_SIZE values should be 'safe' except that, obviously, no JSON
|
||||
* document can ever span 0 or 1 byte and that very large values would create memory allocation issues.
|
||||
*/
|
||||
static constexpr size_t MINIMAL_BATCH_SIZE = 32;
|
||||
|
||||
/**
|
||||
* It is wasteful to allocate memory for tiny documents (e.g., 4 bytes).
|
||||
*/
|
||||
static constexpr size_t MINIMAL_DOCUMENT_CAPACITY = 32;
|
||||
|
||||
/**
|
||||
* A persistent document parser.
|
||||
*
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
#ifndef SIMDJSON_SERIALIZATION_INL_H
|
||||
#define SIMDJSON_SERIALIZATION_INL_H
|
||||
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/dom/serialization.h"
|
||||
|
||||
#include <cinttypes>
|
||||
#include <type_traits>
|
||||
#include "simdjson/dom/parser.h"
|
||||
#include "simdjson/internal/tape_type.h"
|
||||
|
||||
namespace simdjson {
|
||||
namespace dom {
|
||||
@@ -41,7 +41,7 @@ struct escape_sequence {
|
||||
* adds the to_chars functions which would do as well, but
|
||||
* we want to support C++11.
|
||||
*/
|
||||
char *fast_itoa(char *output, int64_t value) noexcept {
|
||||
static char *fast_itoa(char *output, int64_t value) noexcept {
|
||||
// This is a standard implementation of itoa.
|
||||
char buffer[20];
|
||||
uint64_t value_positive;
|
||||
@@ -83,7 +83,7 @@ char *fast_itoa(char *output, int64_t value) noexcept {
|
||||
* adds the to_chars functions which would do as well, but
|
||||
* we want to support C++11.
|
||||
*/
|
||||
char *fast_itoa(char *output, uint64_t value) noexcept {
|
||||
static char *fast_itoa(char *output, uint64_t value) noexcept {
|
||||
// This is a standard implementation of itoa.
|
||||
char buffer[20];
|
||||
const char *const end_buffer = buffer + 20;
|
||||
@@ -100,6 +100,8 @@ char *fast_itoa(char *output, uint64_t value) noexcept {
|
||||
std::memcpy(output, write_pointer, len);
|
||||
return output + len;
|
||||
}
|
||||
|
||||
|
||||
} // anonymous namespace
|
||||
namespace internal {
|
||||
|
||||
@@ -107,19 +109,22 @@ namespace internal {
|
||||
* Minifier/formatter code.
|
||||
**/
|
||||
|
||||
simdjson_inline void mini_formatter::number(uint64_t x) {
|
||||
template<class formatter>
|
||||
simdjson_inline void base_formatter<formatter>::number(uint64_t x) {
|
||||
char number_buffer[24];
|
||||
char *newp = fast_itoa(number_buffer, x);
|
||||
buffer.insert(buffer.end(), number_buffer, newp);
|
||||
}
|
||||
|
||||
simdjson_inline void mini_formatter::number(int64_t x) {
|
||||
template<class formatter>
|
||||
simdjson_inline void base_formatter<formatter>::number(int64_t x) {
|
||||
char number_buffer[24];
|
||||
char *newp = fast_itoa(number_buffer, x);
|
||||
buffer.insert(buffer.end(), number_buffer, newp);
|
||||
}
|
||||
|
||||
simdjson_inline void mini_formatter::number(double x) {
|
||||
template<class formatter>
|
||||
simdjson_inline void base_formatter<formatter>::number(double x) {
|
||||
char number_buffer[24];
|
||||
// Currently, passing the nullptr to the second argument is
|
||||
// safe because our implementation does not check the second
|
||||
@@ -128,31 +133,51 @@ simdjson_inline void mini_formatter::number(double x) {
|
||||
buffer.insert(buffer.end(), number_buffer, newp);
|
||||
}
|
||||
|
||||
simdjson_inline void mini_formatter::start_array() { one_char('['); }
|
||||
simdjson_inline void mini_formatter::end_array() { one_char(']'); }
|
||||
simdjson_inline void mini_formatter::start_object() { one_char('{'); }
|
||||
simdjson_inline void mini_formatter::end_object() { one_char('}'); }
|
||||
simdjson_inline void mini_formatter::comma() { one_char(','); }
|
||||
template<class formatter>
|
||||
simdjson_inline void base_formatter<formatter>::start_array() { one_char('['); }
|
||||
|
||||
|
||||
simdjson_inline void mini_formatter::true_atom() {
|
||||
template<class formatter>
|
||||
simdjson_inline void base_formatter<formatter>::end_array() { one_char(']'); }
|
||||
|
||||
template<class formatter>
|
||||
simdjson_inline void base_formatter<formatter>::start_object() { one_char('{'); }
|
||||
|
||||
template<class formatter>
|
||||
simdjson_inline void base_formatter<formatter>::end_object() { one_char('}'); }
|
||||
|
||||
template<class formatter>
|
||||
simdjson_inline void base_formatter<formatter>::comma() { one_char(','); }
|
||||
|
||||
template<class formatter>
|
||||
simdjson_inline void base_formatter<formatter>::true_atom() {
|
||||
const char * s = "true";
|
||||
buffer.insert(buffer.end(), s, s + 4);
|
||||
}
|
||||
simdjson_inline void mini_formatter::false_atom() {
|
||||
|
||||
template<class formatter>
|
||||
simdjson_inline void base_formatter<formatter>::false_atom() {
|
||||
const char * s = "false";
|
||||
buffer.insert(buffer.end(), s, s + 5);
|
||||
}
|
||||
simdjson_inline void mini_formatter::null_atom() {
|
||||
|
||||
template<class formatter>
|
||||
simdjson_inline void base_formatter<formatter>::null_atom() {
|
||||
const char * s = "null";
|
||||
buffer.insert(buffer.end(), s, s + 4);
|
||||
}
|
||||
simdjson_inline void mini_formatter::one_char(char c) { buffer.push_back(c); }
|
||||
simdjson_inline void mini_formatter::key(std::string_view unescaped) {
|
||||
|
||||
template<class formatter>
|
||||
simdjson_inline void base_formatter<formatter>::one_char(char c) { buffer.push_back(c); }
|
||||
|
||||
template<class formatter>
|
||||
simdjson_inline void base_formatter<formatter>::key(std::string_view unescaped) {
|
||||
string(unescaped);
|
||||
one_char(':');
|
||||
}
|
||||
simdjson_inline void mini_formatter::string(std::string_view unescaped) {
|
||||
|
||||
template<class formatter>
|
||||
simdjson_inline void base_formatter<formatter>::string(std::string_view unescaped) {
|
||||
one_char('\"');
|
||||
size_t i = 0;
|
||||
// Fast path for the case where we have no control character, no ", and no backslash.
|
||||
@@ -231,14 +256,46 @@ simdjson_inline void mini_formatter::string(std::string_view unescaped) {
|
||||
one_char('\"');
|
||||
}
|
||||
|
||||
inline void mini_formatter::clear() {
|
||||
|
||||
template<class formatter>
|
||||
inline void base_formatter<formatter>::clear() {
|
||||
buffer.clear();
|
||||
}
|
||||
|
||||
simdjson_inline std::string_view mini_formatter::str() const {
|
||||
template<class formatter>
|
||||
simdjson_inline std::string_view base_formatter<formatter>::str() const {
|
||||
return std::string_view(buffer.data(), buffer.size());
|
||||
}
|
||||
|
||||
simdjson_inline void mini_formatter::print_newline() {
|
||||
return;
|
||||
}
|
||||
|
||||
simdjson_inline void mini_formatter::print_indents(size_t depth) {
|
||||
(void)depth;
|
||||
return;
|
||||
}
|
||||
|
||||
simdjson_inline void mini_formatter::print_space() {
|
||||
return;
|
||||
}
|
||||
|
||||
simdjson_inline void pretty_formatter::print_newline() {
|
||||
one_char('\n');
|
||||
}
|
||||
|
||||
simdjson_inline void pretty_formatter::print_indents(size_t depth) {
|
||||
if(this->indent_step <= 0) {
|
||||
return;
|
||||
}
|
||||
for(size_t i = 0; i < this->indent_step * depth; i++) {
|
||||
one_char(' ');
|
||||
}
|
||||
}
|
||||
|
||||
simdjson_inline void pretty_formatter::print_space() {
|
||||
one_char(' ');
|
||||
}
|
||||
|
||||
/***
|
||||
* String building code.
|
||||
@@ -258,11 +315,16 @@ inline void string_builder<serializer>::append(simdjson::dom::element value) {
|
||||
// print commas after each value
|
||||
if (after_value) {
|
||||
format.comma();
|
||||
format.print_newline();
|
||||
}
|
||||
|
||||
format.print_indents(depth);
|
||||
|
||||
// If we are in an object, print the next key and :, and skip to the next
|
||||
// value.
|
||||
if (is_object[depth]) {
|
||||
format.key(iter.get_string_view());
|
||||
format.print_space();
|
||||
iter.json_index++;
|
||||
}
|
||||
switch (iter.tape_ref_type()) {
|
||||
@@ -291,6 +353,7 @@ inline void string_builder<serializer>::append(simdjson::dom::element value) {
|
||||
|
||||
is_object[depth] = false;
|
||||
after_value = false;
|
||||
format.print_newline();
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -318,6 +381,7 @@ inline void string_builder<serializer>::append(simdjson::dom::element value) {
|
||||
|
||||
is_object[depth] = true;
|
||||
after_value = false;
|
||||
format.print_newline();
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -362,17 +426,21 @@ inline void string_builder<serializer>::append(simdjson::dom::element value) {
|
||||
// Handle multiple ends in a row
|
||||
while (depth != 0 && (iter.tape_ref_type() == tape_type::END_ARRAY ||
|
||||
iter.tape_ref_type() == tape_type::END_OBJECT)) {
|
||||
format.print_newline();
|
||||
depth--;
|
||||
format.print_indents(depth);
|
||||
if (iter.tape_ref_type() == tape_type::END_ARRAY) {
|
||||
format.end_array();
|
||||
} else {
|
||||
format.end_object();
|
||||
}
|
||||
depth--;
|
||||
iter.json_index++;
|
||||
}
|
||||
|
||||
// Stop when we're at depth 0
|
||||
} while (depth != 0);
|
||||
|
||||
format.print_newline();
|
||||
}
|
||||
|
||||
template <class serializer>
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
#ifndef SIMDJSON_SERIALIZATION_H
|
||||
#define SIMDJSON_SERIALIZATION_H
|
||||
|
||||
#include "simdjson/common_defs.h"
|
||||
#include "simdjson/dom/document.h"
|
||||
#include "simdjson/error.h"
|
||||
#include "simdjson/internal/dom_parser_implementation.h"
|
||||
#include "simdjson/internal/tape_ref.h"
|
||||
#include "simdjson/padded_string.h"
|
||||
#include "simdjson/portability.h"
|
||||
#include "simdjson/dom/base.h"
|
||||
#include "simdjson/dom/element.h"
|
||||
#include "simdjson/dom/object.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
namespace simdjson {
|
||||
@@ -19,50 +16,9 @@ namespace simdjson {
|
||||
*/
|
||||
namespace internal {
|
||||
|
||||
class mini_formatter;
|
||||
|
||||
/**
|
||||
* @private The string_builder template allows us to construct
|
||||
* a string from a document element. It is parametrized
|
||||
* by a "formatter" which handles the details. Thus
|
||||
* the string_builder template could support both minification
|
||||
* and prettification, and various other tradeoffs.
|
||||
*/
|
||||
template <class formatter = mini_formatter>
|
||||
class string_builder {
|
||||
template<class formatter>
|
||||
class base_formatter {
|
||||
public:
|
||||
/** Construct an initially empty builder, would print the empty string **/
|
||||
string_builder() = default;
|
||||
/** Append an element to the builder (to be printed) **/
|
||||
inline void append(simdjson::dom::element value);
|
||||
/** Append an array to the builder (to be printed) **/
|
||||
inline void append(simdjson::dom::array value);
|
||||
/** Append an object to the builder (to be printed) **/
|
||||
inline void append(simdjson::dom::object value);
|
||||
/** Reset the builder (so that it would print the empty string) **/
|
||||
simdjson_inline void clear();
|
||||
/**
|
||||
* Get access to the string. The string_view is owned by the builder
|
||||
* and it is invalid to use it after the string_builder has been
|
||||
* destroyed.
|
||||
* However you can make a copy of the string_view on memory that you
|
||||
* own.
|
||||
*/
|
||||
simdjson_inline std::string_view str() const;
|
||||
/** Append a key_value_pair to the builder (to be printed) **/
|
||||
simdjson_inline void append(simdjson::dom::key_value_pair value);
|
||||
private:
|
||||
formatter format{};
|
||||
};
|
||||
|
||||
/**
|
||||
* @private This is the class that we expect to use with the string_builder
|
||||
* template. It tries to produce a compact version of the JSON element
|
||||
* as quickly as possible.
|
||||
*/
|
||||
class mini_formatter {
|
||||
public:
|
||||
mini_formatter() = default;
|
||||
/** Add a comma **/
|
||||
simdjson_inline void comma();
|
||||
/** Start an array, prints [ **/
|
||||
@@ -97,14 +53,88 @@ public:
|
||||
**/
|
||||
simdjson_inline std::string_view str() const;
|
||||
|
||||
private:
|
||||
// implementation details (subject to change)
|
||||
/** Prints one character **/
|
||||
simdjson_inline void one_char(char c);
|
||||
|
||||
simdjson_inline void call_print_newline() {
|
||||
this->print_newline();
|
||||
}
|
||||
|
||||
simdjson_inline void call_print_indents(size_t depth) {
|
||||
this->print_indents(depth);
|
||||
}
|
||||
|
||||
simdjson_inline void call_print_space() {
|
||||
this->print_space();
|
||||
}
|
||||
|
||||
protected:
|
||||
// implementation details (subject to change)
|
||||
/** Backing buffer **/
|
||||
std::vector<char> buffer{}; // not ideal!
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* @private This is the class that we expect to use with the string_builder
|
||||
* template. It tries to produce a compact version of the JSON element
|
||||
* as quickly as possible.
|
||||
*/
|
||||
class mini_formatter : public base_formatter<mini_formatter> {
|
||||
public:
|
||||
simdjson_inline void print_newline();
|
||||
|
||||
simdjson_inline void print_indents(size_t depth);
|
||||
|
||||
simdjson_inline void print_space();
|
||||
};
|
||||
|
||||
class pretty_formatter : public base_formatter<pretty_formatter> {
|
||||
public:
|
||||
simdjson_inline void print_newline();
|
||||
|
||||
simdjson_inline void print_indents(size_t depth);
|
||||
|
||||
simdjson_inline void print_space();
|
||||
|
||||
protected:
|
||||
int indent_step = 4;
|
||||
};
|
||||
|
||||
/**
|
||||
* @private The string_builder template allows us to construct
|
||||
* a string from a document element. It is parametrized
|
||||
* by a "formatter" which handles the details. Thus
|
||||
* the string_builder template could support both minification
|
||||
* and prettification, and various other tradeoffs.
|
||||
*/
|
||||
template <class formatter = mini_formatter>
|
||||
class string_builder {
|
||||
public:
|
||||
/** Construct an initially empty builder, would print the empty string **/
|
||||
string_builder() = default;
|
||||
/** Append an element to the builder (to be printed) **/
|
||||
inline void append(simdjson::dom::element value);
|
||||
/** Append an array to the builder (to be printed) **/
|
||||
inline void append(simdjson::dom::array value);
|
||||
/** Append an object to the builder (to be printed) **/
|
||||
inline void append(simdjson::dom::object value);
|
||||
/** Reset the builder (so that it would print the empty string) **/
|
||||
simdjson_inline void clear();
|
||||
/**
|
||||
* Get access to the string. The string_view is owned by the builder
|
||||
* and it is invalid to use it after the string_builder has been
|
||||
* destroyed.
|
||||
* However you can make a copy of the string_view on memory that you
|
||||
* own.
|
||||
*/
|
||||
simdjson_inline std::string_view str() const;
|
||||
/** Append a key_value_pair to the builder (to be printed) **/
|
||||
simdjson_inline void append(simdjson::dom::key_value_pair value);
|
||||
private:
|
||||
formatter format{};
|
||||
};
|
||||
|
||||
} // internal
|
||||
|
||||
namespace dom {
|
||||
@@ -212,6 +242,38 @@ std::string minify(simdjson_result<T> x) {
|
||||
}
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Prettifies a JSON element or document, printing the valid JSON with indentation.
|
||||
*
|
||||
* dom::parser parser;
|
||||
* element doc = parser.parse(" [ 1 , 2 , 3 ] "_padded);
|
||||
*
|
||||
* // Prints:
|
||||
* // {
|
||||
* // [
|
||||
* // 1,
|
||||
* // 2,
|
||||
* // 3
|
||||
* // ]
|
||||
* // }
|
||||
* cout << prettify(doc) << endl;
|
||||
*
|
||||
*/
|
||||
template <class T>
|
||||
std::string prettify(T x) {
|
||||
simdjson::internal::string_builder<simdjson::internal::pretty_formatter> sb;
|
||||
sb.append(x);
|
||||
std::string_view answer = sb.str();
|
||||
return std::string(answer.data(), answer.size());
|
||||
}
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
template <class T>
|
||||
std::string prettify(simdjson_result<T> x) {
|
||||
if (x.error()) { throw simdjson_error(x.error()); }
|
||||
return to_string(x.value());
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace simdjson
|
||||
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
#ifndef SIMDJSON_INLINE_ERROR_H
|
||||
#define SIMDJSON_INLINE_ERROR_H
|
||||
#ifndef SIMDJSON_ERROR_INL_H
|
||||
#define SIMDJSON_ERROR_INL_H
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include "simdjson/error.h"
|
||||
|
||||
namespace simdjson {
|
||||
@@ -182,4 +181,4 @@ simdjson_inline simdjson_result<T>::simdjson_result() noexcept
|
||||
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_INLINE_ERROR_H
|
||||
#endif // SIMDJSON_ERROR_INL_H
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#ifndef SIMDJSON_ERROR_H
|
||||
#define SIMDJSON_ERROR_H
|
||||
|
||||
#include "simdjson/common_defs.h"
|
||||
#include "simdjson/base.h"
|
||||
#include <string>
|
||||
|
||||
namespace simdjson {
|
||||
|
||||
@@ -1,30 +1,8 @@
|
||||
#ifndef SIMDJSON_FALLBACK_H
|
||||
#define SIMDJSON_FALLBACK_H
|
||||
|
||||
#include "simdjson/implementation-base.h"
|
||||
|
||||
#if SIMDJSON_IMPLEMENTATION_FALLBACK
|
||||
|
||||
namespace simdjson {
|
||||
/**
|
||||
* Fallback implementation (runs on any machine).
|
||||
*/
|
||||
namespace fallback {
|
||||
} // namespace fallback
|
||||
} // namespace simdjson
|
||||
|
||||
#include "simdjson/fallback/implementation.h"
|
||||
|
||||
#include "simdjson/fallback/begin.h"
|
||||
|
||||
// Declarations
|
||||
#include "simdjson/generic/dom_parser_implementation.h"
|
||||
#include "simdjson/fallback/bitmanipulation.h"
|
||||
#include "simdjson/generic/jsoncharutils.h"
|
||||
#include "simdjson/generic/atomparsing.h"
|
||||
#include "simdjson/fallback/stringparsing.h"
|
||||
#include "simdjson/fallback/numberparsing.h"
|
||||
#include "simdjson/generic/amalgamated.h"
|
||||
#include "simdjson/fallback/end.h"
|
||||
|
||||
#endif // SIMDJSON_IMPLEMENTATION_FALLBACK
|
||||
#endif // SIMDJSON_FALLBACK_H
|
||||
#endif // SIMDJSON_FALLBACK_H
|
||||
@@ -0,0 +1,26 @@
|
||||
#ifndef SIMDJSON_FALLBACK_BASE_H
|
||||
#define SIMDJSON_FALLBACK_BASE_H
|
||||
|
||||
#ifndef SIMDJSON_AMALGAMATED
|
||||
#include "simdjson/base.h"
|
||||
#endif // SIMDJSON_AMALGAMATED
|
||||
|
||||
namespace simdjson {
|
||||
/**
|
||||
* Fallback implementation (runs on any machine).
|
||||
*/
|
||||
namespace fallback {
|
||||
|
||||
class implementation;
|
||||
|
||||
} // namespace fallback
|
||||
} // namespace simdjson
|
||||
|
||||
#ifndef SIMDJSON_AMALGAMATED
|
||||
// If we're editing one of the files in this directory, begin the implementation!
|
||||
#ifndef SIMDJSON_IMPLEMENTATION
|
||||
#include "simdjson/fallback/begin.h"
|
||||
#endif
|
||||
#endif // SIMDJSON_AMALGAMATED
|
||||
|
||||
#endif // SIMDJSON_FALLBACK_BASE_H
|
||||
@@ -1 +1,5 @@
|
||||
#define SIMDJSON_IMPLEMENTATION fallback
|
||||
#include "simdjson/fallback/base.h"
|
||||
#include "simdjson/fallback/bitmanipulation.h"
|
||||
#include "simdjson/fallback/stringparsing_defs.h"
|
||||
#include "simdjson/fallback/numberparsing_defs.h"
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
#ifndef SIMDJSON_FALLBACK_BITMANIPULATION_H
|
||||
#define SIMDJSON_FALLBACK_BITMANIPULATION_H
|
||||
|
||||
#include "simdjson/base.h"
|
||||
#include <limits>
|
||||
#include "simdjson/fallback/base.h"
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
|
||||
@@ -1,16 +1,13 @@
|
||||
#ifndef SIMDJSON_FALLBACK_IMPLEMENTATION_H
|
||||
#define SIMDJSON_FALLBACK_IMPLEMENTATION_H
|
||||
|
||||
#ifndef SIMDJSON_AMALGAMATED
|
||||
#include "simdjson/implementation.h"
|
||||
#endif // SIMDJSON_AMALGAMATED
|
||||
|
||||
namespace simdjson {
|
||||
namespace fallback {
|
||||
|
||||
namespace {
|
||||
using namespace simdjson;
|
||||
using namespace simdjson::dom;
|
||||
}
|
||||
|
||||
/**
|
||||
* @private
|
||||
*/
|
||||
@@ -24,7 +21,7 @@ public:
|
||||
simdjson_warn_unused error_code create_dom_parser_implementation(
|
||||
size_t capacity,
|
||||
size_t max_length,
|
||||
std::unique_ptr<internal::dom_parser_implementation>& dst
|
||||
std::unique_ptr<simdjson::internal::dom_parser_implementation>& dst
|
||||
) const noexcept final;
|
||||
simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
|
||||
simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
#ifndef SIMDJSON_FALLBACK_NUMBERPARSING_H
|
||||
#define SIMDJSON_FALLBACK_NUMBERPARSING_H
|
||||
|
||||
#ifdef JSON_TEST_NUMBERS // for unit testing
|
||||
void found_invalid_number(const uint8_t *buf);
|
||||
void found_integer(int64_t result, const uint8_t *buf);
|
||||
void found_unsigned_integer(uint64_t result, const uint8_t *buf);
|
||||
void found_float(double result, const uint8_t *buf);
|
||||
#endif
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
namespace {
|
||||
// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
|
||||
static simdjson_inline uint32_t parse_eight_digits_unrolled(const char *chars) {
|
||||
uint64_t val;
|
||||
memcpy(&val, chars, sizeof(uint64_t));
|
||||
val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8;
|
||||
val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16;
|
||||
return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32);
|
||||
}
|
||||
static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
|
||||
return parse_eight_digits_unrolled(reinterpret_cast<const char *>(chars));
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
} // namespace simdjson
|
||||
|
||||
#define SIMDJSON_SWAR_NUMBER_PARSING 1
|
||||
|
||||
#include "simdjson/generic/numberparsing.h"
|
||||
|
||||
#endif // SIMDJSON_FALLBACK_NUMBERPARSING_H
|
||||
@@ -0,0 +1,79 @@
|
||||
#ifndef SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H
|
||||
#define SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H
|
||||
|
||||
#include "simdjson/fallback/base.h"
|
||||
|
||||
#ifndef SIMDJSON_AMALGAMATED
|
||||
#include "simdjson/internal/numberparsing_tables.h"
|
||||
#endif // SIMDJSON_AMALGAMATED
|
||||
|
||||
#ifdef JSON_TEST_NUMBERS // for unit testing
|
||||
void found_invalid_number(const uint8_t *buf);
|
||||
void found_integer(int64_t result, const uint8_t *buf);
|
||||
void found_unsigned_integer(uint64_t result, const uint8_t *buf);
|
||||
void found_float(double result, const uint8_t *buf);
|
||||
#endif
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
namespace numberparsing {
|
||||
|
||||
// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
|
||||
/** @private */
|
||||
static simdjson_inline uint32_t parse_eight_digits_unrolled(const char *chars) {
|
||||
uint64_t val;
|
||||
memcpy(&val, chars, sizeof(uint64_t));
|
||||
val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8;
|
||||
val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16;
|
||||
return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32);
|
||||
}
|
||||
|
||||
/** @private */
|
||||
static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
|
||||
return parse_eight_digits_unrolled(reinterpret_cast<const char *>(chars));
|
||||
}
|
||||
|
||||
#if SIMDJSON_IS_32BITS // _umul128 for x86, arm
|
||||
// this is a slow emulation routine for 32-bit
|
||||
//
|
||||
static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) {
|
||||
return x * (uint64_t)y;
|
||||
}
|
||||
static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) {
|
||||
uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd);
|
||||
uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd);
|
||||
uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32));
|
||||
uint64_t adbc_carry = !!(adbc < ad);
|
||||
uint64_t lo = bd + (adbc << 32);
|
||||
*hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) +
|
||||
(adbc_carry << 32) + !!(lo < bd);
|
||||
return lo;
|
||||
}
|
||||
#endif
|
||||
|
||||
/** @private */
|
||||
simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) {
|
||||
internal::value128 answer;
|
||||
#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS
|
||||
#ifdef _M_ARM64
|
||||
// ARM64 has native support for 64-bit multiplications, no need to emultate
|
||||
answer.high = __umulh(value1, value2);
|
||||
answer.low = value1 * value2;
|
||||
#else
|
||||
answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64
|
||||
#endif // _M_ARM64
|
||||
#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS
|
||||
__uint128_t r = (static_cast<__uint128_t>(value1)) * value2;
|
||||
answer.low = uint64_t(r);
|
||||
answer.high = uint64_t(r >> 64);
|
||||
#endif
|
||||
return answer;
|
||||
}
|
||||
|
||||
} // namespace numberparsing
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
} // namespace simdjson
|
||||
|
||||
#define SIMDJSON_SWAR_NUMBER_PARSING 1
|
||||
|
||||
#endif // SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H
|
||||
@@ -0,0 +1,8 @@
|
||||
#ifndef SIMDJSON_FALLBACK_ONDEMAND_H
|
||||
#define SIMDJSON_FALLBACK_ONDEMAND_H
|
||||
|
||||
#include "simdjson/fallback/begin.h"
|
||||
#include "simdjson/generic/ondemand/amalgamated.h"
|
||||
#include "simdjson/fallback/end.h"
|
||||
|
||||
#endif // SIMDJSON_FALLBACK_ONDEMAND_H
|
||||
@@ -0,0 +1,8 @@
|
||||
#ifndef SIMDJSON_FALLBACK_SINGLESTAGE_H
|
||||
#define SIMDJSON_FALLBACK_SINGLESTAGE_H
|
||||
|
||||
#include "simdjson/fallback/begin.h"
|
||||
#include "simdjson/generic/singlestage/amalgamated.h"
|
||||
#include "simdjson/fallback/end.h"
|
||||
|
||||
#endif // SIMDJSON_FALLBACK_SINGLESTAGE_H
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user