mirror of
https://github.com/simdjson/simdjson
synced 2026-06-08 17:27:07 +00:00
Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3620e9d151 | |||
| d017cd7ca4 | |||
| d9d1ff5856 | |||
| eb8f2bce14 | |||
| 2a4ff73468 | |||
| 77fc2b8447 | |||
| ba8b66a633 | |||
| 66eec5feaf | |||
| 3964f3e5d2 |
@@ -1,23 +0,0 @@
|
||||
name: Ubuntu 24.04 CI (GCC 13)
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
ubuntu-build:
|
||||
if: >-
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip github]')
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: dependencies/.cache
|
||||
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
|
||||
- name: Use cmake
|
||||
run: |
|
||||
mkdir build &&
|
||||
cd build &&
|
||||
CXX=g++-13 cmake -DSIMDJSON_DEVELOPER_MODE=ON .. &&
|
||||
cmake --build . &&
|
||||
ctest --output-on-failure -LE explicitonly -j
|
||||
@@ -0,0 +1,24 @@
|
||||
name: Ubuntu 24.04 CI
|
||||
|
||||
on: [push, pull_request]
|
||||
jobs:
|
||||
ubuntu-build:
|
||||
if: >-
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip github]')
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
shared: [ON, OFF]
|
||||
cxx: [g++-13, clang++-16]
|
||||
sanitizer: [ON, OFF]
|
||||
steps:
|
||||
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
|
||||
- name: Prepare
|
||||
run: cmake -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_SANITIZE=${{matrix.sanitizer}} -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build
|
||||
env:
|
||||
CXX: ${{matrix.cxx}}
|
||||
- name: Build
|
||||
run: cmake --build build -j=2
|
||||
- name: Test
|
||||
run: ctest --output-on-failure --test-dir build
|
||||
+1
-1
@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
|
||||
project(
|
||||
simdjson
|
||||
# The version number is modified by tools/release.py
|
||||
VERSION 3.9.3
|
||||
VERSION 3.9.4
|
||||
DESCRIPTION "Parsing gigabytes of JSON per second"
|
||||
HOMEPAGE_URL "https://simdjson.org/"
|
||||
LANGUAGES CXX C
|
||||
|
||||
@@ -38,7 +38,7 @@ PROJECT_NAME = simdjson
|
||||
# could be handy for archiving the generated documentation or if some version
|
||||
# control system is used.
|
||||
|
||||
PROJECT_NUMBER = "3.9.3"
|
||||
PROJECT_NUMBER = "3.9.4"
|
||||
|
||||
# Using the PROJECT_BRIEF tag one can provide an optional one line description
|
||||
# for a project that appears at the top of each page and should give viewer a
|
||||
|
||||
+11
-4
@@ -64,20 +64,23 @@ into your project. Then include it in your project with:
|
||||
using namespace simdjson; // optional
|
||||
```
|
||||
|
||||
You can compile with:
|
||||
Under most systems, you can compile with:
|
||||
|
||||
```
|
||||
c++ myproject.cpp simdjson.cpp
|
||||
```
|
||||
|
||||
Note:
|
||||
- We recommend that you use simdjson by copying the single-header `simdjson.h` file along with the source file `simdjson.cpp` directly in your project, as they are part of [every release](https://github.com/simdjson/simdjson/releases) as assets. In this manner, you only have to compile `simdjson.cpp` as any other source file: it works well in every development environment. However, you may also use simdjson as a git submodule ([example](https://github.com/simdjson/cmakedemo)), using FetchContent ([example](https://github.com/simdjson/cmake_demo_single_file)), with ExternalProject_Add ([example](https://github.com/simdjson/cmakedemo_externalproject)) or with CPM ([example](https://github.com/cpm-cmake/CPM.cmake/tree/master/examples/simdjson)).
|
||||
- Users on macOS and other platforms where default compilers do not provide C++11 compliant by default should request it with the appropriate flag (e.g., `c++ -std=c++11 myproject.cpp simdjson.cpp`).
|
||||
- The library relies on [runtime CPU detection](implementation-selection.md): avoid specifying an architecture at compile time (e.g., `-march-native`) if you want your binaries to run everywhere.
|
||||
|
||||
Using simdjson with package managers
|
||||
------------------
|
||||
|
||||
You can install the simdjson library on your system or in your project using multiple package managers such as MSYS2, the conan package manager, vcpkg, brew, the apt package manager (debian-based Linux systems), the FreeBSD package manager (FreeBSD), and so on. [Visit our wiki for more details](https://github.com/simdjson/simdjson/wiki/Installing-simdjson-with-a-package-manager).
|
||||
You can install the simdjson library on your system or in your project using multiple package managers such as MSYS2, the conan package manager, vcpkg, brew, the apt package manager (debian-based Linux systems), the FreeBSD package manager (FreeBSD), and so on. E.g., [we provide an complete example with vcpkg](https://github.com/simdjson/simdjson-vcpkg) that works under Windows. [Visit our wiki for more details](https://github.com/simdjson/simdjson/wiki/Installing-simdjson-with-a-package-manager).
|
||||
|
||||
|
||||
|
||||
The following Linux distributions provide simdjson packages: Alpine, RedHat, Rocky Linux, Debian, Fedora, and Ubuntu.
|
||||
|
||||
@@ -156,7 +159,8 @@ For efficiency reasons, simdjson requires a string with a few bytes (`simdjson::
|
||||
at the end, these bytes may be read but their content does not affect the parsing. In practice,
|
||||
it means that the JSON inputs should be stored in a memory region with `simdjson::SIMDJSON_PADDING`
|
||||
extra bytes at the end. You do not have to set these bytes to specific values though you may
|
||||
want to if you want to avoid runtime warnings with some sanitizers.
|
||||
want to if you want to avoid runtime warnings with some sanitizers. Advanced users may want to
|
||||
read the section Free Padding in [our performance notes](performance.md).
|
||||
|
||||
The simdjson library offers a tree-like [API](https://en.wikipedia.org/wiki/API), which you can
|
||||
access by creating a `ondemand::parser` and calling the `iterate()` method. The iterate method
|
||||
@@ -567,7 +571,7 @@ support for users who avoid exceptions. See [the simdjson error handling documen
|
||||
auto json = R"( { "test":{ "val1":1, "val2":2 } } )"_padded;
|
||||
auto doc = parser.iterate(json);
|
||||
size_t count = doc.count_fields(); // requires simdjson 1.0 or better
|
||||
std::cout << "Number of fields: " << new_count << std::endl; // Prints "Number of fields: 1"
|
||||
std::cout << "Number of fields: " << count << std::endl; // Prints "Number of fields: 1"
|
||||
```
|
||||
Similarly to `count_elements`, you should not let an object instance go out of scope before consuming it after calling
|
||||
the `count_fields` method. If you access an object inside a document, you can use the `count_fields` method as follow.
|
||||
@@ -2596,6 +2600,7 @@ Performance tips
|
||||
--------
|
||||
|
||||
|
||||
- Read [our performance notes](performance.md) for advanced topics.
|
||||
- The On Demand front-end works best when doing a single pass over the input: avoid calling `count_elements`, `rewind` and similar methods.
|
||||
- If you are familiar with assembly language, you may use the online tool godbolt to explore the compiled code. The following example may work: [https://godbolt.org/z/xE4GWs573](https://godbolt.org/z/xE4GWs573).
|
||||
- Given a field `field` in an object, calling `field.key()` is often faster than `field.unescaped_key()` so if you do not need an unescaped `std::string_view` instance, prefer `field.key()`. Similarly, we expect `field.escaped_key()` to be faster than `field.unescaped_key()` even though both return a `std::string_view` instance.
|
||||
@@ -2618,6 +2623,8 @@ Performance tips
|
||||
```
|
||||
- To better understand the operation of your On Demand parser, and whether it is performing as well as you think it should be, there is a logger feature built in to simdjson! To use it, define the pre-processor directive `SIMDJSON_VERBOSE_LOGGING` prior to including the `simdjson.h` header, which enables logging in simdjson. Run your code. It may generate a lot of logging output; adding printouts from your application that show each section may be helpful. The log's output will show step-by-step information on state, buffer pointer position, depth, and key retrieval status. Importantly, unless `SIMDJSON_VERBOSE_LOGGING` is defined, logging is entirely disabled and thus carries no overhead.
|
||||
|
||||
|
||||
|
||||
Further reading
|
||||
--------
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ testing and get the best performance.
|
||||
* [Number parsing](#number-parsing)
|
||||
* [Visual Studio](#visual-studio)
|
||||
* [Power Usage and Downclocking](#power-usage-and-downclocking)
|
||||
* [Free Padding](#free-padding)
|
||||
|
||||
|
||||
NDEBUG directive
|
||||
@@ -179,3 +180,110 @@ The simdjson library does not generally make use of heavy 256-bit instructions.
|
||||
the macro `SIMDJSON_AVX512_ALLOWED` to `0` in C++ prior to importing the headers.
|
||||
|
||||
You may still be worried about which SIMD instruction set is used by simdjson. Thankfully, [you can always determine and change which architecture-specific implementation is used](implementation-selection.md) by simdjson. Thus even if your CPU supports AVX2, you do not need to use AVX2. You are in control.
|
||||
|
||||
|
||||
Free Padding
|
||||
-------
|
||||
|
||||
For performance reasons, the simdjson library requires that the JSON input contain at least
|
||||
`simdjson::SIMDJSON_PADDING` bytes at the end of the stream. The value `simdjson::SIMDJSON_PADDING` is
|
||||
small (e.g., 64 bytes). On modern systems, you can safely read beyond an allocated buffers,
|
||||
as long as you remain within an allocated page. Pages on modern systems span at least 4 kilobytes,
|
||||
but can be significantly larger. E.g., Apple systems favour pages spanning 16 kilobytes.
|
||||
|
||||
In effect, it means that you can almost always read a few bytes beyond your current buffer---without
|
||||
allocating extra memory. However, tools such as valgrind or memory sanitizers will flag such behavior as unsafe.
|
||||
Nevertheless, you can still make sure of this capability in your code if you are an expert
|
||||
programmer and you are willing to silence sanitizer warnings. The following code provides
|
||||
a portable example.
|
||||
|
||||
|
||||
The conditional compilation checks for the `_MSC_VER` macro (indicating Microsoft Visual Studio)
|
||||
and includes platform-specific headers accordingly.
|
||||
The `page_size()` function determines the default size of a memory page in bytes on the system.
|
||||
On Windows (when `_WIN32` is defined), it uses `GetSystemInfo()` to retrieve system information and obtain the page size.
|
||||
On other platforms (non-Windows), it uses `sysconf(_SC_PAGESIZE)` to get the page size.
|
||||
The function returns the page size.
|
||||
The `need_allocation()` function checks whether the buffer (given by `buf`) plus the specified length (`len`) is near a page boundary.
|
||||
If the buffer extends beyond the current page when padded by `simdjson::SIMDJSON_PADDING`, it returns true, indicating that reallocation is needed.
|
||||
Otherwise, it returns false.
|
||||
The `get_padded_string_view()` creates a `padded_string_view` from the input buffer.
|
||||
If reallocation is needed (unlikely case), it allocates a new padded_string and assigns it to `jsonbuffer`.
|
||||
Otherwise (very likely), it creates a `padded_string_view` directly from the buffer.
|
||||
The `simdjson::SIMDJSON_PADDING` ensures that there is additional padding for parsing efficiency.
|
||||
The calling code just needs to provide `jsonbuffer` (an instance of `simdjson::padded_string`)
|
||||
and pass `get_padded_string_view(buf, len, jsonbuffer)` to `parser.iterate`. Most of the time,
|
||||
this code will not allocate new memory.
|
||||
|
||||
|
||||
```cpp
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
#include <sysinfoapi.h>
|
||||
#else
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
#include "simdjson.h"
|
||||
#include <cstdio>
|
||||
|
||||
// Returns the default size of the page in bytes on this system.
|
||||
long page_size() {
|
||||
#ifdef _WIN32
|
||||
SYSTEM_INFO sysInfo;
|
||||
GetSystemInfo(&sysInfo);
|
||||
long pagesize = sysInfo.dwPageSize;
|
||||
#else
|
||||
long pagesize = sysconf(_SC_PAGESIZE);
|
||||
#endif
|
||||
return pagesize;
|
||||
}
|
||||
|
||||
// Returns true if the buffer + len + simdjson::SIMDJSON_PADDING crosses the
|
||||
// page boundary.
|
||||
bool need_allocation(const char *buf, size_t len) {
|
||||
return ((reinterpret_cast<uintptr_t>(buf + len - 1) % page_size()) <
|
||||
simdjson::SIMDJSON_PADDING);
|
||||
}
|
||||
|
||||
simdjson::padded_string_view
|
||||
get_padded_string_view(const char *buf, size_t len,
|
||||
simdjson::padded_string &jsonbuffer) {
|
||||
if (need_allocation(buf, len)) { // unlikely case
|
||||
jsonbuffer = simdjson::padded_string(buf, len);
|
||||
return jsonbuffer;
|
||||
} else { // no reallcation needed (very likely)
|
||||
return simdjson::padded_string_view(buf, len,
|
||||
len + simdjson::SIMDJSON_PADDING);
|
||||
}
|
||||
}
|
||||
|
||||
int main() {
|
||||
printf("page_size: %ld\n", page_size());
|
||||
const char *jsonpoiner = R"(
|
||||
{
|
||||
"key": "value"
|
||||
}
|
||||
)";
|
||||
size_t len = strlen(jsonpoiner);
|
||||
simdjson::padded_string jsonbuffer; // only allocate if needed
|
||||
simdjson::ondemand::parser parser;
|
||||
simdjson::ondemand::document doc;
|
||||
simdjson::error_code error =
|
||||
parser.iterate(get_padded_string_view(jsonpoiner, len, jsonbuffer))
|
||||
.get(doc);
|
||||
if (error) {
|
||||
printf("error: %s\n", simdjson::error_message(error));
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
std::string_view value;
|
||||
error = doc["key"].get_string().get(value);
|
||||
if (error) {
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
printf("Value: \"%.*s\"\n", (int)value.size(), value.data());
|
||||
if (value != "value") {
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
```
|
||||
@@ -6,13 +6,13 @@
|
||||
// Distributed under the Boost Software License, Version 1.0.
|
||||
// (See accompanying file LICENSE.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
|
||||
// #pragma once // We remove #pragma once here as it generates a warning in some cases. We rely on the include guard.
|
||||
#pragma once
|
||||
|
||||
#ifndef NONSTD_SV_LITE_H_INCLUDED
|
||||
#define NONSTD_SV_LITE_H_INCLUDED
|
||||
|
||||
#define string_view_lite_MAJOR 1
|
||||
#define string_view_lite_MINOR 7
|
||||
#define string_view_lite_MINOR 8
|
||||
#define string_view_lite_PATCH 0
|
||||
|
||||
#define string_view_lite_VERSION nssv_STRINGIFY(string_view_lite_MAJOR) "." nssv_STRINGIFY(string_view_lite_MINOR) "." nssv_STRINGIFY(string_view_lite_PATCH)
|
||||
@@ -134,6 +134,8 @@
|
||||
|
||||
#if nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace nonstd {
|
||||
|
||||
template< class CharT, class Traits, class Allocator = std::allocator<CharT> >
|
||||
|
||||
@@ -11,6 +11,9 @@
|
||||
#include <strings.h>
|
||||
#endif
|
||||
|
||||
// We are using size_t without namespace std:: throughout the project
|
||||
using std::size_t;
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define SIMDJSON_VISUAL_STUDIO 1
|
||||
/**
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
#define SIMDJSON_SIMDJSON_VERSION_H
|
||||
|
||||
/** The version of simdjson being used (major.minor.revision) */
|
||||
#define SIMDJSON_VERSION "3.9.3"
|
||||
#define SIMDJSON_VERSION "3.9.4"
|
||||
|
||||
namespace simdjson {
|
||||
enum {
|
||||
@@ -19,7 +19,7 @@ enum {
|
||||
/**
|
||||
* The revision (major.minor.REVISION) of simdjson being used.
|
||||
*/
|
||||
SIMDJSON_VERSION_REVISION = 3
|
||||
SIMDJSON_VERSION_REVISION = 4
|
||||
};
|
||||
} // namespace simdjson
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/* auto-generated on 2024-05-30 10:52:38 -0400. Do not edit! */
|
||||
/* auto-generated on 2024-06-11 14:08:20 -0400. Do not edit! */
|
||||
/* including simdjson.cpp: */
|
||||
/* begin file simdjson.cpp */
|
||||
#define SIMDJSON_SRC_SIMDJSON_CPP
|
||||
@@ -84,6 +84,9 @@
|
||||
#include <strings.h>
|
||||
#endif
|
||||
|
||||
// We are using size_t without namespace std:: throughout the project
|
||||
using std::size_t;
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define SIMDJSON_VISUAL_STUDIO 1
|
||||
/**
|
||||
@@ -527,13 +530,13 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
|
||||
// Distributed under the Boost Software License, Version 1.0.
|
||||
// (See accompanying file LICENSE.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
|
||||
// #pragma once // We remove #pragma once here as it generates a warning in some cases. We rely on the include guard.
|
||||
#pragma once
|
||||
|
||||
#ifndef NONSTD_SV_LITE_H_INCLUDED
|
||||
#define NONSTD_SV_LITE_H_INCLUDED
|
||||
|
||||
#define string_view_lite_MAJOR 1
|
||||
#define string_view_lite_MINOR 7
|
||||
#define string_view_lite_MINOR 8
|
||||
#define string_view_lite_PATCH 0
|
||||
|
||||
#define string_view_lite_VERSION nssv_STRINGIFY(string_view_lite_MAJOR) "." nssv_STRINGIFY(string_view_lite_MINOR) "." nssv_STRINGIFY(string_view_lite_PATCH)
|
||||
@@ -655,6 +658,8 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
|
||||
|
||||
#if nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace nonstd {
|
||||
|
||||
template< class CharT, class Traits, class Allocator = std::allocator<CharT> >
|
||||
|
||||
+10
-5
@@ -1,4 +1,4 @@
|
||||
/* auto-generated on 2024-05-30 10:52:38 -0400. Do not edit! */
|
||||
/* auto-generated on 2024-06-11 14:08:20 -0400. Do not edit! */
|
||||
/* including simdjson.h: */
|
||||
/* begin file simdjson.h */
|
||||
#ifndef SIMDJSON_H
|
||||
@@ -104,6 +104,9 @@
|
||||
#include <strings.h>
|
||||
#endif
|
||||
|
||||
// We are using size_t without namespace std:: throughout the project
|
||||
using std::size_t;
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define SIMDJSON_VISUAL_STUDIO 1
|
||||
/**
|
||||
@@ -547,13 +550,13 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
|
||||
// Distributed under the Boost Software License, Version 1.0.
|
||||
// (See accompanying file LICENSE.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
|
||||
// #pragma once // We remove #pragma once here as it generates a warning in some cases. We rely on the include guard.
|
||||
#pragma once
|
||||
|
||||
#ifndef NONSTD_SV_LITE_H_INCLUDED
|
||||
#define NONSTD_SV_LITE_H_INCLUDED
|
||||
|
||||
#define string_view_lite_MAJOR 1
|
||||
#define string_view_lite_MINOR 7
|
||||
#define string_view_lite_MINOR 8
|
||||
#define string_view_lite_PATCH 0
|
||||
|
||||
#define string_view_lite_VERSION nssv_STRINGIFY(string_view_lite_MAJOR) "." nssv_STRINGIFY(string_view_lite_MINOR) "." nssv_STRINGIFY(string_view_lite_PATCH)
|
||||
@@ -675,6 +678,8 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
|
||||
|
||||
#if nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace nonstd {
|
||||
|
||||
template< class CharT, class Traits, class Allocator = std::allocator<CharT> >
|
||||
@@ -2346,7 +2351,7 @@ namespace std {
|
||||
#define SIMDJSON_SIMDJSON_VERSION_H
|
||||
|
||||
/** The version of simdjson being used (major.minor.revision) */
|
||||
#define SIMDJSON_VERSION "3.9.3"
|
||||
#define SIMDJSON_VERSION "3.9.4"
|
||||
|
||||
namespace simdjson {
|
||||
enum {
|
||||
@@ -2361,7 +2366,7 @@ enum {
|
||||
/**
|
||||
* The revision (major.minor.REVISION) of simdjson being used.
|
||||
*/
|
||||
SIMDJSON_VERSION_REVISION = 3
|
||||
SIMDJSON_VERSION_REVISION = 4
|
||||
};
|
||||
} // namespace simdjson
|
||||
|
||||
|
||||
@@ -28,6 +28,9 @@ add_cpp_test(ondemand_to_string LABELS ondemand acceptance per_impl
|
||||
add_cpp_test(ondemand_twitter_tests LABELS ondemand acceptance per_implementation)
|
||||
add_cpp_test(ondemand_wrong_type_error_tests LABELS ondemand acceptance per_implementation)
|
||||
add_cpp_test(ondemand_iterate_many_csv LABELS ondemand acceptance per_implementation)
|
||||
if(NOT SIMDJSON_SANITIZE)
|
||||
add_cpp_test(ondemand_cacheline LABELS ondemand acceptance per_implementation)
|
||||
endif()
|
||||
|
||||
if(HAVE_POSIX_FORK AND HAVE_POSIX_WAIT) # assert tests use fork and wait, which aren't on MSVC
|
||||
add_cpp_test(ondemand_assert_out_of_order_values LABELS assert per_implementation explicitonly ondemand)
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
#include <sysinfoapi.h>
|
||||
#else
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
#include "simdjson.h"
|
||||
#include <cstdio>
|
||||
// Returns the default size of the page in bytes on this system.
|
||||
long page_size() {
|
||||
#ifdef _WIN32
|
||||
SYSTEM_INFO sysInfo;
|
||||
GetSystemInfo(&sysInfo);
|
||||
long pagesize = sysInfo.dwPageSize;
|
||||
#else
|
||||
long pagesize = sysconf(_SC_PAGESIZE);
|
||||
#endif
|
||||
return pagesize;
|
||||
}
|
||||
|
||||
// Returns true if the buffer + len + simdjson::SIMDJSON_PADDING crosses the
|
||||
// page boundary.
|
||||
bool need_allocation(const char *buf, size_t len) {
|
||||
return ((reinterpret_cast<uintptr_t>(buf + len - 1) % page_size()) <
|
||||
simdjson::SIMDJSON_PADDING);
|
||||
}
|
||||
|
||||
simdjson::padded_string_view
|
||||
get_padded_string_view(const char *buf, size_t len,
|
||||
simdjson::padded_string &jsonbuffer) {
|
||||
if (need_allocation(buf, len)) { // unlikely case
|
||||
jsonbuffer = simdjson::padded_string(buf, len);
|
||||
return jsonbuffer;
|
||||
} else { // no reallcation needed (very likely)
|
||||
return simdjson::padded_string_view(buf, len,
|
||||
len + simdjson::SIMDJSON_PADDING);
|
||||
}
|
||||
}
|
||||
|
||||
int main() {
|
||||
printf("page_size: %ld\n", page_size());
|
||||
const char *jsonpoiner = R"(
|
||||
{
|
||||
"key": "value"
|
||||
}
|
||||
)";
|
||||
size_t len = strlen(jsonpoiner);
|
||||
simdjson::padded_string jsonbuffer; // only allocate if needed
|
||||
simdjson::ondemand::parser parser;
|
||||
simdjson::ondemand::document doc;
|
||||
simdjson::error_code error =
|
||||
parser.iterate(get_padded_string_view(jsonpoiner, len, jsonbuffer))
|
||||
.get(doc);
|
||||
if (error) {
|
||||
printf("error: %s\n", simdjson::error_message(error));
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
std::string_view value;
|
||||
error = doc["key"].get_string().get(value);
|
||||
if (error) {
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
printf("Value: \"%.*s\"\n", (int)value.size(), value.data());
|
||||
if (value != "value") {
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
@@ -5,6 +5,22 @@ using namespace simdjson;
|
||||
|
||||
namespace misc_tests {
|
||||
using namespace std;
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
// user reported an asan error:
|
||||
bool issue2199() {
|
||||
TEST_START();
|
||||
static constexpr std::string_view kJsonString = R"( { "name": "name", "version": 100, } )";
|
||||
try {
|
||||
simdjson::padded_string buffer{kJsonString};
|
||||
simdjson::ondemand::parser parser;
|
||||
simdjson::ondemand::document document = parser.iterate(buffer);
|
||||
(void)document;
|
||||
} catch (simdjson::simdjson_error& /*error*/) {
|
||||
std::cerr << "Caught simdjson_error" << std::endl;
|
||||
}
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
#endif
|
||||
bool issue1981_success() {
|
||||
auto error_phrase = R"(false)"_padded;
|
||||
TEST_START();
|
||||
@@ -532,7 +548,7 @@ namespace misc_tests {
|
||||
string_view token;
|
||||
ASSERT_SUCCESS(o["value"].raw_json_token().get(token));
|
||||
ASSERT_EQUAL(token, "12321323213213213213213213213211223");
|
||||
return true;
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
simdjson_warn_unused bool big_integer_in_string() {
|
||||
TEST_START();
|
||||
@@ -545,7 +561,7 @@ namespace misc_tests {
|
||||
string_view token;
|
||||
ASSERT_SUCCESS(o["value"].raw_json_token().get(token));
|
||||
ASSERT_EQUAL(token, "\"12321323213213213213213213213211223\"");
|
||||
return true;
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
simdjson_warn_unused bool test_raw_json_token(string_view json, string_view expected_token, int expected_start_index = 0) {
|
||||
string title("'");
|
||||
@@ -558,7 +574,7 @@ namespace misc_tests {
|
||||
ASSERT_EQUAL( token, expected_token );
|
||||
// Validate the text is inside the original buffer
|
||||
ASSERT_EQUAL( reinterpret_cast<const void*>(token.data()), reinterpret_cast<const void*>(&json_padded.data()[expected_start_index]));
|
||||
return true;
|
||||
TEST_SUCCEED();
|
||||
}));
|
||||
|
||||
// Test values
|
||||
@@ -576,10 +592,9 @@ namespace misc_tests {
|
||||
// Validate the text is inside the original buffer
|
||||
// Adjust for the {"a":
|
||||
ASSERT_EQUAL( reinterpret_cast<const void*>(token.data()), reinterpret_cast<const void*>(&json_padded.data()[5+expected_start_index]));
|
||||
return true;
|
||||
TEST_SUCCEED();
|
||||
}));
|
||||
|
||||
return true;
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
bool raw_json_token() {
|
||||
@@ -605,6 +620,9 @@ namespace misc_tests {
|
||||
|
||||
bool run() {
|
||||
return
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
issue2199() &&
|
||||
#endif
|
||||
skipbom() &&
|
||||
issue1981_success() &&
|
||||
issue1981_failure() &&
|
||||
|
||||
Reference in New Issue
Block a user