mirror of
https://github.com/simdjson/simdjson
synced 2026-06-08 17:27:07 +00:00
adding memory-file mapping to Windows + better doc (#2676)
* adding memory-file mapping to Windows. * making memory-file mapping optional under windows, as it is fragile * removing non-ascii * saving. * bumping up * take 2
This commit is contained in:
@@ -13,18 +13,20 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- {gen: Visual Studio 17 2022, arch: Win32, shared: ON, build_type: Release}
|
||||
- {gen: Visual Studio 17 2022, arch: Win32, shared: OFF, build_type: Release}
|
||||
- {gen: Visual Studio 17 2022, arch: x64, shared: ON, build_type: Release}
|
||||
- {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Debug}
|
||||
- {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Release}
|
||||
- {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: RelWithDebInfo}
|
||||
- {gen: Visual Studio 17 2022, arch: Win32, shared: ON, build_type: Release, memory_map: OFF}
|
||||
- {gen: Visual Studio 17 2022, arch: Win32, shared: OFF, build_type: Release, memory_map: OFF}
|
||||
- {gen: Visual Studio 17 2022, arch: x64, shared: ON, build_type: Release, memory_map: OFF}
|
||||
- {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Debug, memory_map: OFF}
|
||||
- {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Release, memory_map: OFF}
|
||||
- {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: RelWithDebInfo, memory_map: OFF}
|
||||
# Exercise the opt-in Windows memory-file mapping path at least once in CI.
|
||||
- {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Release, memory_map: ON}
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Configure
|
||||
run: |
|
||||
cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build
|
||||
cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -DSIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=${{matrix.memory_map}} -B build
|
||||
- name: Build Debug
|
||||
run: cmake --build build --config ${{matrix.build_type}} --verbose
|
||||
- name: Run tests
|
||||
|
||||
@@ -75,6 +75,41 @@ if(SIMDJSON_DEVELOPMENT_CHECKS)
|
||||
)
|
||||
endif()
|
||||
|
||||
# padded_memory_map is always available on POSIX. On Windows it is disabled
|
||||
# by default because it depends on the `CreateFileMapping2` / `MapViewOfFile3`
|
||||
# APIs, which require Windows 10 version 1803 or later and are exported via
|
||||
# onecore.lib rather than the default kernel32.lib. Turn this option ON to
|
||||
# opt into the feature on Windows; simdjson will then set the appropriate
|
||||
# Windows version macros and link onecore, so everything that links
|
||||
# simdjson picks up both the compile-time declarations and the import
|
||||
# library automatically. The option is a no-op on POSIX (where the feature
|
||||
# is unconditionally enabled).
|
||||
option(SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS
|
||||
"Enable simdjson::padded_memory_map on Windows (requires Windows 10 \
|
||||
version 1803 or later). Always enabled on POSIX." OFF)
|
||||
if(SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS)
|
||||
simdjson_add_props(
|
||||
target_compile_definitions PUBLIC
|
||||
SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1
|
||||
)
|
||||
if(WIN32)
|
||||
# Raise the Windows version floor so that <windows.h> declares the
|
||||
# modern memory-mapping APIs, and link the import library that
|
||||
# actually exports them. _WIN32_WINNT / WINVER / NTDDI_VERSION together
|
||||
# tell <sdkddkver.h> which APIs to light up.
|
||||
simdjson_add_props(
|
||||
target_compile_definitions PUBLIC
|
||||
_WIN32_WINNT=0x0A00
|
||||
WINVER=0x0A00
|
||||
NTDDI_VERSION=0x0A000006 # NTDDI_WIN10_RS5, Windows 10 version 1809
|
||||
)
|
||||
simdjson_add_props(
|
||||
target_link_libraries PUBLIC
|
||||
onecore
|
||||
)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(is_top_project)
|
||||
option(SIMDJSON_INSTALL "Enable target install" ON)
|
||||
option(SIMDJSON_DEVELOPER_MODE "Enable targets for developing simdjson" OFF)
|
||||
|
||||
+28
-3
@@ -269,10 +269,35 @@ ondemand::document doc = parser.iterate(simdjson::pad(json));
|
||||
We recommend against creating many `std::string` or many `std::padded_string` instances in your application to store your JSON data.
|
||||
Consider reusing the same buffers and limiting memory allocations.
|
||||
|
||||
**Memory-file mapping (non-Windows).** You can use memory-file mapping to create a `simdjson::padded_string_view`
|
||||
from a file on disk:
|
||||
**Memory-file mapping.** You can use `simdjson::padded_memory_map` to create a
|
||||
`simdjson::padded_string_view` from a file on disk. On POSIX systems (Linux,
|
||||
macOS, BSD, ...) it uses `mmap` for true zero-copy access and is always
|
||||
available. On Windows it is an **opt-in** feature because it relies on the
|
||||
`CreateFileMapping2` / `MapViewOfFile3` APIs (Windows 10, version 1803 or
|
||||
later) which are exported from `onecore.lib` rather than the default
|
||||
`kernel32.lib`. To enable it, you must satisfy **all** of the following:
|
||||
|
||||
1. Building simdjson with `-DSIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=ON`, or
|
||||
defining `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1` and raising
|
||||
`NTDDI_VERSION` to at least `NTDDI_WIN10_RS4` (Windows 10, version 1803)
|
||||
and linking `onecore.lib` manually if you are consuming simdjson as a
|
||||
pre-built library.
|
||||
2. `#include <windows.h>` before including simdjson, in every translation
|
||||
unit that uses `padded_memory_map`.
|
||||
|
||||
The Windows implementation then uses `CreateFileMapping2` / `MapViewOfFile3`
|
||||
for true zero-copy access whenever possible, with a transparent
|
||||
buffered-read fallback for files that end too close to a page boundary.
|
||||
|
||||
The availability of the class can be tested with the preprocessor macro
|
||||
`SIMDJSON_HAS_PADDED_MEMORY_MAP`.
|
||||
|
||||
```cpp
|
||||
#ifdef _WIN32
|
||||
#include <windows.h> // Must come BEFORE <simdjson.h> on Windows
|
||||
#endif
|
||||
#include "simdjson.h"
|
||||
// ...
|
||||
simdjson::padded_memory_map map(myfilename);
|
||||
if (!map.is_valid()) { /* handle error */ }
|
||||
simdjson::padded_string_view view = map.view();
|
||||
@@ -305,7 +330,7 @@ Some users may want to browse code along with the compiled assembly:
|
||||
| `simdjson::pad(std::string&)` | Adds padding if needed | Returns `padded_string_view` pointing to the (possibly resized) string | References original string | Recommended to silence sanitizers when using `std::string`. |
|
||||
| `padded_string(data, length)` or `padded_string(std::string)` | Automatic (copies into padded buffer) | Explicit copy into owned padded buffer | Owned by `padded_string` | Safe when you want full ownership and padding guaranteed. |
|
||||
| `padded_string_view` (manual) | User guarantees `SIMDJSON_PADDING` extra bytes after the viewed length | User provides pointer + length + capacity | Non-owning view | Low-level; requires careful buffer management. |
|
||||
| Memory-mapped file (`padded_memory_map`) | Automatic via mapping (non-Windows only) | Creates view with sufficient padding | Non-owning (tied to map lifetime) | Advanced; efficient for large files on Linux/macOS/etc. |
|
||||
| Memory-mapped file (`padded_memory_map`) | Automatic via mapping / padded read | Creates view with sufficient padding | Non-owning (tied to map lifetime) | Always available on POSIX (zero-copy `mmap`). On Windows, opt-in via `-DSIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=ON` (requires Windows 10 1803+ and links `onecore.lib`) and `#include <windows.h>` before simdjson; uses `CreateFileMapping2` + `MapViewOfFile3`. |
|
||||
|
||||
|
||||
Documents are iterators
|
||||
|
||||
+23
-3
@@ -127,11 +127,31 @@ codepage, and they may call SetFileApisToOEM accordingly.
|
||||
|
||||
|
||||
**Advanced feature:**
|
||||
On non-Windows systems, you can use memory-file mapping to create a `simdjson::padded_string_view`
|
||||
from a file on disk.
|
||||
You can use `simdjson::padded_memory_map` to create a `simdjson::padded_string_view`
|
||||
from a file on disk without copying the file contents into your own buffer.
|
||||
On POSIX systems (Linux, macOS, BSD, ...) it uses `mmap` for true zero-copy
|
||||
access. On Windows it is available as an **opt-in** feature and requires:
|
||||
|
||||
1. Building simdjson with `-DSIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=ON`, or
|
||||
defining `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1` and raising
|
||||
`NTDDI_VERSION` to at least `NTDDI_WIN10_RS4` (Windows 10, version 1803)
|
||||
and linking `onecore.lib` manually if you are consuming simdjson as a
|
||||
pre-built library.
|
||||
2. `#include <windows.h>` before `#include "simdjson.h"` in every
|
||||
translation unit where you want to use `padded_memory_map`.
|
||||
|
||||
When enabled on Windows, the implementation uses `CreateFileMapping2` and
|
||||
`MapViewOfFile3` for true zero-copy mapping whenever the file does not end
|
||||
within `SIMDJSON_PADDING` bytes of a page boundary; otherwise it falls back
|
||||
to reading the file into a padded heap buffer. If those requirements are
|
||||
not met, the class is not declared and the code below will fail to compile.
|
||||
|
||||
```cpp
|
||||
// if the macro _WIN32 is defined, this will not work since we do not support Windows
|
||||
#ifdef _WIN32
|
||||
#include <windows.h> // Must come BEFORE <simdjson.h> on Windows
|
||||
#endif
|
||||
#include "simdjson.h"
|
||||
// ...
|
||||
simdjson::padded_memory_map map(TWITTER_JSON);
|
||||
if (!map.is_valid()) { /* handle error */ }
|
||||
simdjson::padded_string_view view = map.view(); // view is usable while padded_memory_map is in scope
|
||||
|
||||
+72
-5
@@ -22,6 +22,7 @@ Contents
|
||||
- [Threads](#threads)
|
||||
- [Support](#support)
|
||||
- [API](#api)
|
||||
- [Streaming directly from a memory-mapped file](#streaming-directly-from-a-memory-mapped-file)
|
||||
- [Use cases](#use-cases)
|
||||
- [Tracking your position](#tracking-your-position)
|
||||
- [Incomplete streams](#incomplete-streams)
|
||||
@@ -156,13 +157,79 @@ for (auto doc : docs) {
|
||||
See [basics.md](basics.md#newline-delimited-json-ndjson-and-json-lines) for an overview of the API.
|
||||
|
||||
|
||||
**Advanced feature:**
|
||||
On non-Windows systems, you can use memory-file mapping to create a `simdjson::padded_string_view`
|
||||
from a file on disk.
|
||||
Streaming directly from a memory-mapped file
|
||||
--------------------------------------------
|
||||
|
||||
When your input is a large NDJSON / JSON-lines file on disk, the most efficient
|
||||
way to feed `iterate_many` is to use `simdjson::padded_memory_map`. It returns
|
||||
a `padded_string_view` with the right amount of trailing padding, so you can
|
||||
hand it straight to `iterate_many` without ever copying the file contents into
|
||||
your own buffer.
|
||||
|
||||
`padded_memory_map` is available on POSIX systems (Linux, macOS, BSD, ...) by
|
||||
default. On Windows it is an **opt-in** feature with the following
|
||||
requirements:
|
||||
|
||||
1. Build simdjson with `-DSIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=ON`, or — if
|
||||
you consume simdjson as a pre-built library — define
|
||||
`SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1`, raise `NTDDI_VERSION` to at
|
||||
least `NTDDI_WIN10_RS4` (`0x0A000005`, Windows 10 version 1803), and
|
||||
add `onecore.lib` to your link line yourself. The Windows
|
||||
implementation uses the modern memory APIs `CreateFileMapping2` /
|
||||
`MapViewOfFile3`, which are available starting with that version of
|
||||
Windows and are exported by `onecore.lib`.
|
||||
2. `#include <windows.h>` before `#include "simdjson.h"` in every
|
||||
translation unit where you want to use `padded_memory_map`. simdjson
|
||||
deliberately does not pull in `<windows.h>` itself, so the class is
|
||||
only declared when the Win32 types are already visible.
|
||||
|
||||
If either requirement is not met on Windows, the `padded_memory_map` class is
|
||||
not declared at all and any code that references it fails to compile with an
|
||||
"unknown identifier" error. The availability of the class can be tested with
|
||||
the macro `SIMDJSON_HAS_PADDED_MEMORY_MAP`.
|
||||
|
||||
On POSIX, `padded_memory_map` uses `mmap` to map the file directly into
|
||||
memory with zero copies. On Windows (when enabled), it uses
|
||||
`CreateFileMapping2` + `MapViewOfFile3` for true zero-copy mapping
|
||||
whenever the file does not end within `SIMDJSON_PADDING` bytes of a page
|
||||
boundary; for those rare cases, it transparently falls back to reading
|
||||
the file into a heap-allocated padded buffer so that the returned view
|
||||
always has `SIMDJSON_PADDING` accessible zero bytes after the file content.
|
||||
|
||||
```cpp
|
||||
#ifdef _WIN32
|
||||
#include <windows.h> // Must come BEFORE <simdjson.h> on Windows
|
||||
#endif
|
||||
#include "simdjson.h"
|
||||
|
||||
// ...
|
||||
|
||||
simdjson::padded_memory_map map("huge_stream.ndjson");
|
||||
if (!map.is_valid()) { /* file missing, unreadable, too large, ... */ return; }
|
||||
|
||||
simdjson::ondemand::parser parser;
|
||||
simdjson::ondemand::document_stream stream;
|
||||
auto error = parser.iterate_many(map.view()).get(stream);
|
||||
if (error) { std::cerr << error << std::endl; return; }
|
||||
|
||||
for (auto doc : stream) {
|
||||
// process each JSON document in the stream
|
||||
std::cout << doc << std::endl;
|
||||
}
|
||||
```
|
||||
|
||||
Important lifetime rule: the `padded_string_view` returned by `map.view()` is
|
||||
only valid while the `padded_memory_map` instance is alive, so keep `map`
|
||||
alive for as long as you are iterating the stream.
|
||||
|
||||
The file must not be modified while the memory map is in use. If you need a
|
||||
fully independent copy of the data, use `simdjson::padded_string::load(...)`
|
||||
instead.
|
||||
|
||||
If you prefer single-document parsing on a memory-mapped file, the same
|
||||
pattern applies to `parser.iterate(...)`:
|
||||
|
||||
```cpp
|
||||
// If the macro _WIN32 is defined, this will not work since we do not support memory-file mapping
|
||||
// under Windows at this time.
|
||||
simdjson::padded_memory_map map(myfilename);
|
||||
if (!map.is_valid()) { /* handle error */ }
|
||||
simdjson::padded_string_view view = map.view(); // view is usable while padded_memory_map is in scope
|
||||
|
||||
+73
-6
@@ -18,6 +18,7 @@ Contents
|
||||
- [How it works](#how-it-works)
|
||||
- [Support](#support)
|
||||
- [API](#api)
|
||||
- [Streaming directly from a memory-mapped file](#streaming-directly-from-a-memory-mapped-file)
|
||||
- [Use cases](#use-cases)
|
||||
- [Tracking your position](#tracking-your-position)
|
||||
- [Incomplete streams](#incomplete-streams)
|
||||
@@ -218,17 +219,83 @@ got full document at 29
|
||||
|
||||
|
||||
|
||||
**Advanced feature:**
|
||||
On non-Windows systems, you can use memory-file mapping to create a `simdjson::padded_string_view`
|
||||
from a file on disk.
|
||||
Streaming directly from a memory-mapped file
|
||||
--------------------------------------------
|
||||
|
||||
When your input is a large NDJSON / JSON-lines file on disk, the most
|
||||
efficient way to feed `parse_many` is to use `simdjson::padded_memory_map`.
|
||||
It returns a `padded_string_view` with the right amount of trailing padding,
|
||||
so you can pass it directly to `parse_many` without copying the file content
|
||||
into your own buffer first.
|
||||
|
||||
`padded_memory_map` is available on POSIX systems (Linux, macOS, BSD, ...) by
|
||||
default. On Windows it is an **opt-in** feature with the following
|
||||
requirements:
|
||||
|
||||
1. Build simdjson with `-DSIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=ON`, or — if
|
||||
you consume simdjson as a pre-built library — define
|
||||
`SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1`, raise `NTDDI_VERSION` to at
|
||||
least `NTDDI_WIN10_RS4` (`0x0A000005`, Windows 10 version 1803), and
|
||||
add `onecore.lib` to your link line yourself. The Windows
|
||||
implementation uses the modern memory APIs `CreateFileMapping2` /
|
||||
`MapViewOfFile3`, which are available starting with that version of
|
||||
Windows and are exported by `onecore.lib`.
|
||||
2. `#include <windows.h>` before `#include "simdjson.h"` in every
|
||||
translation unit where you want to use `padded_memory_map`. simdjson
|
||||
deliberately does not pull in `<windows.h>` itself, so the class is
|
||||
only declared when the Win32 types are already visible.
|
||||
|
||||
If either requirement is not met on Windows, the `padded_memory_map` class is
|
||||
not declared at all and any code that references it fails to compile with an
|
||||
"unknown identifier" error. The availability of the class can be tested with
|
||||
the macro `SIMDJSON_HAS_PADDED_MEMORY_MAP`.
|
||||
|
||||
On POSIX, `padded_memory_map` uses `mmap` to map the file directly into
|
||||
memory with zero copies. On Windows (when enabled), it uses
|
||||
`CreateFileMapping2` + `MapViewOfFile3` for true zero-copy mapping
|
||||
whenever the file does not end within `SIMDJSON_PADDING` bytes of a page
|
||||
boundary; for those rare cases, it transparently falls back to reading
|
||||
the file into a heap-allocated padded buffer so that the returned view
|
||||
always has `SIMDJSON_PADDING` accessible zero bytes after the file content.
|
||||
|
||||
```cpp
|
||||
#ifdef _WIN32
|
||||
#include <windows.h> // Must come BEFORE <simdjson.h> on Windows
|
||||
#endif
|
||||
#include "simdjson.h"
|
||||
|
||||
// ...
|
||||
|
||||
simdjson::padded_memory_map map("huge_stream.ndjson");
|
||||
if (!map.is_valid()) { /* file missing, unreadable, too large, ... */ return; }
|
||||
|
||||
simdjson::dom::parser parser;
|
||||
simdjson::dom::document_stream stream;
|
||||
auto error = parser.parse_many(map.view()).get(stream);
|
||||
if (error) { std::cerr << error << std::endl; return; }
|
||||
|
||||
for (auto doc : stream) {
|
||||
// process each JSON document in the stream
|
||||
std::cout << doc << std::endl;
|
||||
}
|
||||
```
|
||||
|
||||
Important lifetime rule: the `padded_string_view` returned by `map.view()` is
|
||||
only valid while the `padded_memory_map` instance is alive, so keep `map`
|
||||
alive for as long as you are iterating the stream.
|
||||
|
||||
The file must not be modified while the memory map is in use. If you need a
|
||||
fully independent copy of the data, use `simdjson::padded_string::load(...)`
|
||||
instead.
|
||||
|
||||
If you prefer single-document parsing on a memory-mapped file, the same
|
||||
pattern applies to `parser.parse(...)`:
|
||||
|
||||
```cpp
|
||||
// If the macro _WIN32 is defined, this will not work since we do not support memory-file mapping
|
||||
// under Windows at this time.
|
||||
simdjson::padded_memory_map map(myfilename);
|
||||
if (!map.is_valid()) { /* handle error */ }
|
||||
simdjson::padded_string_view view = map.view(); // view is usable while padded_memory_map is in scope
|
||||
ondemand::document doc = parser.iterate(view); // parse the JSON
|
||||
simdjson::dom::element doc = parser.parse(view); // parse the JSON
|
||||
```
|
||||
|
||||
Incomplete streams
|
||||
|
||||
@@ -181,6 +181,9 @@ inline simdjson_result<document_stream> parser::parse_many(const std::string &s,
|
||||
inline simdjson_result<document_stream> parser::parse_many(const padded_string &s, size_t batch_size) noexcept {
|
||||
return parse_many(s.data(), s.length(), batch_size);
|
||||
}
|
||||
inline simdjson_result<document_stream> parser::parse_many(const padded_string_view &v, size_t batch_size) noexcept {
|
||||
return parse_many(v.data(), v.length(), batch_size);
|
||||
}
|
||||
|
||||
inline simdjson_result<document_stream> parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) noexcept {
|
||||
if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }
|
||||
@@ -217,6 +220,9 @@ inline simdjson_result<document_stream> parser::parse_many(const std::string &s,
|
||||
inline simdjson_result<document_stream> parser::parse_many(const padded_string &s, size_t batch_size, stream_format format) noexcept {
|
||||
return parse_many(s.data(), s.length(), batch_size, format);
|
||||
}
|
||||
inline simdjson_result<document_stream> parser::parse_many(const padded_string_view &v, size_t batch_size, stream_format format) noexcept {
|
||||
return parse_many(v.data(), v.length(), batch_size, format);
|
||||
}
|
||||
|
||||
simdjson_inline size_t parser::capacity() const noexcept {
|
||||
return implementation ? implementation->capacity() : 0;
|
||||
|
||||
@@ -490,6 +490,16 @@ public:
|
||||
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
|
||||
inline simdjson_result<document_stream> parse_many(const padded_string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept;
|
||||
inline simdjson_result<document_stream> parse_many(const padded_string &&s, size_t batch_size) = delete;// unsafe
|
||||
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size)
|
||||
*
|
||||
* Because padded_string_view guarantees SIMDJSON_PADDING trailing bytes, this
|
||||
* overload is safe to use with buffers that the caller owns elsewhere (for
|
||||
* example, a padded_memory_map), with no extra copy. Without this overload,
|
||||
* passing a padded_string_view would silently bind to the padded_string
|
||||
* overload via an implicit conversion, allocating and copying the input, and
|
||||
* — because that temporary is destroyed at the end of the full-expression —
|
||||
* leaving the returned document_stream pointing at freed memory. */
|
||||
inline simdjson_result<document_stream> parse_many(const padded_string_view &v, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept;
|
||||
|
||||
/** @private We do not want to allow implicit conversion from C string to std::string. */
|
||||
simdjson_result<document_stream> parse_many(const char *buf, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept = delete;
|
||||
@@ -510,6 +520,8 @@ public:
|
||||
inline simdjson_result<document_stream> parse_many(const std::string &s, size_t batch_size, stream_format format) noexcept;
|
||||
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */
|
||||
inline simdjson_result<document_stream> parse_many(const padded_string &s, size_t batch_size, stream_format format) noexcept;
|
||||
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */
|
||||
inline simdjson_result<document_stream> parse_many(const padded_string_view &v, size_t batch_size, stream_format format) noexcept;
|
||||
|
||||
/**
|
||||
* Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length
|
||||
|
||||
@@ -10,13 +10,18 @@
|
||||
#include <climits>
|
||||
#include <cwchar>
|
||||
|
||||
#ifndef _WIN32
|
||||
#if SIMDJSON_HAS_UNISTD_H
|
||||
#include <fcntl.h>
|
||||
#include <stdio.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
// On Windows, `padded_memory_map` (when it is enabled) depends on types and
|
||||
// functions declared in <windows.h>. We deliberately do NOT include that
|
||||
// header here: users of simdjson who want `padded_memory_map` on Windows
|
||||
// must include <windows.h> themselves *before* including this header. See
|
||||
// padded_string.h for the detection logic.
|
||||
|
||||
namespace simdjson {
|
||||
namespace internal {
|
||||
@@ -385,7 +390,9 @@ inline bool padded_string_builder::reserve(size_t additional) noexcept {
|
||||
}
|
||||
|
||||
|
||||
#ifndef _WIN32
|
||||
#if SIMDJSON_HAS_PADDED_MEMORY_MAP
|
||||
|
||||
#if SIMDJSON_HAS_UNISTD_H
|
||||
simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept {
|
||||
|
||||
int fd = open(filename, O_RDONLY);
|
||||
@@ -421,7 +428,132 @@ simdjson_inline padded_memory_map::~padded_memory_map() noexcept {
|
||||
munmap(const_cast<char *>(data), size + simdjson::SIMDJSON_PADDING);
|
||||
}
|
||||
}
|
||||
#elif defined(_WIN32)
|
||||
// Windows zero-copy implementation using placeholder virtual memory.
|
||||
//
|
||||
// We use the modern Windows memory APIs (VirtualAlloc2, CreateFileMapping2,
|
||||
// MapViewOfFile3 — available since Windows 10 1803) to map the file into a
|
||||
// contiguous virtual address range that includes at least SIMDJSON_PADDING
|
||||
// zero bytes after the file content, with no data copies.
|
||||
//
|
||||
// Strategy:
|
||||
// 1. If rounding the file size up to the allocation granularity already
|
||||
// exceeds file_size + SIMDJSON_PADDING, the OS page zero-fill provides
|
||||
// the padding and we use a simple MapViewOfFile3 call.
|
||||
// 2. Otherwise we reserve a contiguous placeholder region via VirtualAlloc2,
|
||||
// split it at the granularity-aligned file boundary, map the file into
|
||||
// the first part, and commit zero pages for the second part (padding).
|
||||
simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept {
|
||||
HANDLE file_handle = ::CreateFileA(
|
||||
filename, GENERIC_READ,
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
|
||||
NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
|
||||
if (file_handle == INVALID_HANDLE_VALUE) {
|
||||
return;
|
||||
}
|
||||
LARGE_INTEGER file_size_li;
|
||||
if (!::GetFileSizeEx(file_handle, &file_size_li) || file_size_li.QuadPart < 0) {
|
||||
::CloseHandle(file_handle);
|
||||
return;
|
||||
}
|
||||
#if SIMDJSON_IS_32BITS
|
||||
if (static_cast<unsigned long long>(file_size_li.QuadPart) >
|
||||
static_cast<unsigned long long>(SIZE_MAX - simdjson::SIMDJSON_PADDING)) {
|
||||
::CloseHandle(file_handle);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
size = static_cast<size_t>(file_size_li.QuadPart);
|
||||
if (size == 0) {
|
||||
::CloseHandle(file_handle);
|
||||
return;
|
||||
}
|
||||
|
||||
HANDLE section = ::CreateFileMapping2(
|
||||
file_handle, NULL, FILE_MAP_READ, PAGE_READONLY,
|
||||
0, 0, NULL, NULL, 0);
|
||||
::CloseHandle(file_handle);
|
||||
if (section == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
SYSTEM_INFO si;
|
||||
::GetSystemInfo(&si);
|
||||
const size_t granularity = static_cast<size_t>(si.dwAllocationGranularity);
|
||||
const size_t file_region = (size + granularity - 1) & ~(granularity - 1);
|
||||
const size_t total_needed = size + simdjson::SIMDJSON_PADDING;
|
||||
|
||||
if (file_region >= total_needed) {
|
||||
// The zero-fill in the last page already covers the padding.
|
||||
PVOID view = ::MapViewOfFile3(
|
||||
section, ::GetCurrentProcess(), NULL, 0, 0,
|
||||
0, PAGE_READONLY, NULL, 0);
|
||||
::CloseHandle(section);
|
||||
if (view != NULL) {
|
||||
data = static_cast<const char *>(view);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// We need extra zero pages beyond the file region. Use the placeholder API
|
||||
// to get a contiguous virtual address range spanning both the file mapping
|
||||
// and the zero-filled padding.
|
||||
const size_t padding_region =
|
||||
((total_needed - file_region) + granularity - 1) & ~(granularity - 1);
|
||||
const size_t reserve_size = file_region + padding_region;
|
||||
|
||||
// Reserve a contiguous placeholder.
|
||||
PVOID placeholder = ::VirtualAlloc2(
|
||||
::GetCurrentProcess(), NULL, reserve_size,
|
||||
MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, NULL, 0);
|
||||
if (placeholder == NULL) {
|
||||
::CloseHandle(section);
|
||||
return;
|
||||
}
|
||||
|
||||
// Split into two placeholders at the file_region boundary.
|
||||
if (!::VirtualFree(placeholder, file_region,
|
||||
MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER)) {
|
||||
::VirtualFree(placeholder, 0, MEM_RELEASE);
|
||||
::CloseHandle(section);
|
||||
return;
|
||||
}
|
||||
|
||||
// Map the file into the first placeholder.
|
||||
PVOID file_view = ::MapViewOfFile3(
|
||||
section, ::GetCurrentProcess(), placeholder, 0, file_region,
|
||||
MEM_REPLACE_PLACEHOLDER, PAGE_READONLY, NULL, 0);
|
||||
::CloseHandle(section);
|
||||
if (file_view == NULL) {
|
||||
::VirtualFree(placeholder, 0, MEM_RELEASE);
|
||||
::VirtualFree(static_cast<char *>(placeholder) + file_region,
|
||||
0, MEM_RELEASE);
|
||||
return;
|
||||
}
|
||||
|
||||
// Commit zero pages in the second placeholder (the padding).
|
||||
void *pad = static_cast<char *>(placeholder) + file_region;
|
||||
PVOID padding_ptr = ::VirtualAlloc2(
|
||||
::GetCurrentProcess(), pad, padding_region,
|
||||
MEM_REPLACE_PLACEHOLDER | MEM_COMMIT, PAGE_READONLY, NULL, 0);
|
||||
if (padding_ptr == NULL) {
|
||||
::UnmapViewOfFile(file_view);
|
||||
::VirtualFree(pad, 0, MEM_RELEASE);
|
||||
return;
|
||||
}
|
||||
|
||||
data = static_cast<const char *>(file_view);
|
||||
padding_view_ = padding_ptr;
|
||||
}
|
||||
|
||||
simdjson_inline padded_memory_map::~padded_memory_map() noexcept {
|
||||
if (data == nullptr) { return; }
|
||||
::UnmapViewOfFile(data);
|
||||
if (padding_view_ != nullptr) {
|
||||
::VirtualFree(padding_view_, 0, MEM_RELEASE);
|
||||
}
|
||||
}
|
||||
#endif // POSIX or _WIN32
|
||||
|
||||
simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noexcept simdjson_lifetime_bound {
|
||||
if(!is_valid()) {
|
||||
@@ -433,7 +565,8 @@ simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noe
|
||||
simdjson_inline bool padded_memory_map::is_valid() const noexcept {
|
||||
return data != nullptr;
|
||||
}
|
||||
#endif // _WIN32
|
||||
|
||||
#endif // SIMDJSON_HAS_PADDED_MEMORY_MAP
|
||||
|
||||
} // namespace simdjson
|
||||
|
||||
|
||||
@@ -277,11 +277,26 @@ inline std::ostream& operator<<(std::ostream& out, const padded_string& s) { ret
|
||||
inline std::ostream& operator<<(std::ostream& out, simdjson_result<padded_string> &s) noexcept(false) { return out << s.value(); }
|
||||
#endif
|
||||
|
||||
|
||||
#ifndef _WIN32
|
||||
#if SIMDJSON_HAS_PADDED_MEMORY_MAP
|
||||
/**
|
||||
* A class representing a memory-mapped file with padding.
|
||||
* It is only available on non-Windows platforms, as Windows has different APIs for memory mapping.
|
||||
*
|
||||
* On POSIX systems (Linux, macOS, BSD, ...), this uses `mmap` to map the file
|
||||
* contents directly into memory, which is efficient for large files (no copy).
|
||||
*
|
||||
* On Windows, this class is disabled by default and must be opted into at
|
||||
* build time by defining `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1`. When
|
||||
* enabled, `<windows.h>` must also be included before `<simdjson.h>` and
|
||||
* the compilation must target Windows 10, version 1803 or later. The
|
||||
* Windows implementation uses the modern memory APIs (`VirtualAlloc2`,
|
||||
* `CreateFileMapping2`, `MapViewOfFile3`) with the placeholder virtual
|
||||
* memory mechanism to always achieve true zero-copy mapping with
|
||||
* contiguous zero-filled padding.
|
||||
*
|
||||
* Either way, the resulting `padded_string_view` carries at least
|
||||
* `SIMDJSON_PADDING` bytes of accessible zero-filled padding after the file
|
||||
* content, so it can be consumed directly by the simdjson parsers (including
|
||||
* `parse_many` / `iterate_many`).
|
||||
*/
|
||||
class padded_memory_map {
|
||||
public:
|
||||
@@ -289,9 +304,11 @@ public:
|
||||
* Create a new padded memory map for the given file.
|
||||
* After creating the memory map, you can call view() to get a padded_string_view of the file content.
|
||||
* The memory map will be automatically released when the padded_memory_map instance is destroyed.
|
||||
* Note that the file content is not copied, so this is efficient for large files. However,
|
||||
* the file must remain unchanged while the memory map is in use. In case of error (e.g., file not found,
|
||||
* permission denied, etc.), the memory map will be invalid and view() will return an empty view.
|
||||
* On POSIX systems, the file content is not copied, so this is efficient for large files.
|
||||
* On Windows, the file is mapped into memory via `MapViewOfFile3` (zero-copy).
|
||||
* In all cases, the file must remain unchanged while the memory map is in use.
|
||||
* In case of error (e.g., file not found, permission denied, etc.), the memory map will be
|
||||
* invalid and view() will return an empty view.
|
||||
* You can check if the memory map is valid by calling is_valid() before using view().
|
||||
*
|
||||
* @param filename the path to the file to memory-map.
|
||||
@@ -328,8 +345,14 @@ private:
|
||||
padded_memory_map &operator=(const padded_memory_map &) = delete;
|
||||
const char *data{nullptr};
|
||||
size_t size{0};
|
||||
#ifdef _WIN32
|
||||
// When the file ends near an allocation-granularity boundary, we use the
|
||||
// placeholder API to append zero-filled padding pages. This pointer tracks
|
||||
// that region so the destructor can release it with VirtualFree.
|
||||
void *padding_view_{nullptr};
|
||||
#endif
|
||||
};
|
||||
#endif // _WIN32
|
||||
#endif // SIMDJSON_HAS_PADDED_MEMORY_MAP
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
#include <cstring> /* memcmp */
|
||||
|
||||
// for page size computation.
|
||||
#if defined(__unix__) || defined(__APPLE__) || defined(__linux__)
|
||||
#if SIMDJSON_HAS_UNISTD_H
|
||||
#include <unistd.h>
|
||||
#if defined(__APPLE__)
|
||||
#include <sys/sysctl.h>
|
||||
@@ -113,7 +113,7 @@ inline uint32_t get_page_size() noexcept {
|
||||
return static_cast<std::uint32_t>(si.dwPageSize);
|
||||
}();
|
||||
return cached;
|
||||
#elif defined(__unix__) || defined(__APPLE__) || defined(__linux__)
|
||||
#elif SIMDJSON_HAS_UNISTD_H
|
||||
static const uint32_t cached = []() -> uint32_t {
|
||||
long page_size = sysconf(_SC_PAGESIZE);
|
||||
if (page_size > 0) {
|
||||
|
||||
@@ -285,5 +285,53 @@ using std::size_t;
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifndef SIMDJSON_HAS_UNISTD_H
|
||||
#if defined(__unix__) || defined(__APPLE__) || defined(__linux__)
|
||||
#define SIMDJSON_HAS_UNISTD_H 1
|
||||
#else
|
||||
#define SIMDJSON_HAS_UNISTD_H 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// padded_memory_map availability.
|
||||
//
|
||||
// On POSIX platforms the class is always available: the implementation uses
|
||||
// `mmap` (and a trailing anonymous page for padding) from <sys/mman.h>.
|
||||
//
|
||||
// On Windows the class is disabled by default and must be explicitly
|
||||
// opted into by defining `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1`. Enabling
|
||||
// it requires:
|
||||
// 1. `<windows.h>` has been included *before* `<simdjson.h>` (so that
|
||||
// this header can see the Win32 types and the `_WINDOWS_` include
|
||||
// guard),
|
||||
// 2. the compilation targets Windows 10, version 1803 or later
|
||||
// (i.e. `NTDDI_VERSION >= NTDDI_WIN10_RS4`, `0x0A000005`). This is
|
||||
// required because the implementation relies on the modern memory
|
||||
// APIs introduced with that version (`CreateFileMapping2` /
|
||||
// `MapViewOfFile3`),
|
||||
// 3. the link step pulls in an import library that exports those APIs,
|
||||
// typically `onecore.lib` (or `mincore.lib`).
|
||||
//
|
||||
// The `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS` CMake option arranges (1)-(3)
|
||||
// automatically when building simdjson with its own CMake. Consumers using
|
||||
// simdjson as a pre-built library are responsible for setting the macro,
|
||||
// the Windows version macros, and the link library themselves.
|
||||
//
|
||||
// If the opt-in conditions are not met on Windows, `padded_memory_map`
|
||||
// simply does not exist — any attempt to use it fails at compile time
|
||||
// with an "unknown identifier" diagnostic rather than silently degrading.
|
||||
//
|
||||
// The SIMDJSON_HAS_PADDED_MEMORY_MAP macro reflects whether the class is
|
||||
// available in the current translation unit. Users may test this macro to
|
||||
// conditionally compile code that depends on padded_memory_map.
|
||||
#ifndef SIMDJSON_HAS_PADDED_MEMORY_MAP
|
||||
#if defined(__unix__) || defined(__APPLE__) || defined(__linux__)
|
||||
#define SIMDJSON_HAS_PADDED_MEMORY_MAP 1
|
||||
#elif defined(_WINDOWS_) && defined(SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS) && SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS
|
||||
#define SIMDJSON_HAS_PADDED_MEMORY_MAP 1
|
||||
#else
|
||||
#define SIMDJSON_HAS_PADDED_MEMORY_MAP 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#endif // SIMDJSON_PORTABILITY_H
|
||||
|
||||
+203
-7
@@ -4356,10 +4356,55 @@ inline std::ostream& operator<<(std::ostream& out, simdjson_result<padded_string
|
||||
#endif
|
||||
|
||||
|
||||
#ifndef _WIN32
|
||||
// padded_memory_map availability.
|
||||
//
|
||||
// On POSIX platforms the class is always available: the implementation uses
|
||||
// `mmap` (and a trailing anonymous page for padding) from <sys/mman.h>.
|
||||
//
|
||||
// On Windows the class is only available when all of the following hold:
|
||||
// 1. <windows.h> has been included *before* <simdjson.h> (so that this
|
||||
// header can see the Win32 types and the `_WINDOWS_` include guard),
|
||||
// 2. the compilation targets Windows 11 or later (NTDDI_VERSION
|
||||
// >= NTDDI_WIN10_CO, 0x0A00000B). This is required because the
|
||||
// implementation relies on the modern memory APIs introduced with
|
||||
// that version (CreateFileMapping2 / MapViewOfFile3).
|
||||
//
|
||||
// If those conditions are not met on Windows, `padded_memory_map` simply
|
||||
// does not exist — any attempt to use it fails at compile time with an
|
||||
// "unknown identifier" diagnostic rather than silently degrading.
|
||||
//
|
||||
// The SIMDJSON_HAS_PADDED_MEMORY_MAP macro reflects whether the class is
|
||||
// available in the current translation unit. Users may test this macro to
|
||||
// conditionally compile code that depends on padded_memory_map.
|
||||
#ifndef SIMDJSON_HAS_PADDED_MEMORY_MAP
|
||||
#if !defined(_WIN32)
|
||||
#define SIMDJSON_HAS_PADDED_MEMORY_MAP 1
|
||||
#elif defined(_WINDOWS_) && defined(NTDDI_VERSION) && (NTDDI_VERSION >= 0x0A00000B /* NTDDI_WIN10_CO — Windows 11 */)
|
||||
#define SIMDJSON_HAS_PADDED_MEMORY_MAP 1
|
||||
#else
|
||||
#define SIMDJSON_HAS_PADDED_MEMORY_MAP 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if SIMDJSON_HAS_PADDED_MEMORY_MAP
|
||||
/**
|
||||
* A class representing a memory-mapped file with padding.
|
||||
* It is only available on non-Windows platforms, as Windows has different APIs for memory mapping.
|
||||
*
|
||||
* On POSIX systems (Linux, macOS, BSD, ...), this uses `mmap` to map the file
|
||||
* contents directly into memory, which is efficient for large files (no copy).
|
||||
*
|
||||
* On Windows, this class is only available when `<windows.h>` is included
|
||||
* before `<simdjson.h>` and the compilation targets Windows 11 or later
|
||||
* (NTDDI_VERSION >= NTDDI_WIN10_CO). The Windows implementation uses the
|
||||
* modern memory APIs (`CreateFileMapping2` / `MapViewOfFile3`) to map the
|
||||
* file with true zero-copy semantics whenever the last page of the file
|
||||
* provides enough trailing zero-fill for SIMDJSON_PADDING bytes; otherwise
|
||||
* it falls back to a heap-allocated padded buffer populated with `ReadFile`.
|
||||
*
|
||||
* Either way, the resulting `padded_string_view` carries at least
|
||||
* `SIMDJSON_PADDING` bytes of accessible zero-filled padding after the file
|
||||
* content, so it can be consumed directly by the simdjson parsers (including
|
||||
* `parse_many` / `iterate_many`).
|
||||
*/
|
||||
class padded_memory_map {
|
||||
public:
|
||||
@@ -4367,9 +4412,12 @@ public:
|
||||
* Create a new padded memory map for the given file.
|
||||
* After creating the memory map, you can call view() to get a padded_string_view of the file content.
|
||||
* The memory map will be automatically released when the padded_memory_map instance is destroyed.
|
||||
* Note that the file content is not copied, so this is efficient for large files. However,
|
||||
* the file must remain unchanged while the memory map is in use. In case of error (e.g., file not found,
|
||||
* permission denied, etc.), the memory map will be invalid and view() will return an empty view.
|
||||
* On POSIX systems, the file content is not copied, so this is efficient for large files.
|
||||
* On Windows, the file is mapped into memory via `MapViewOfFile3` whenever possible
|
||||
* (zero-copy) and otherwise read into a heap-allocated padded buffer.
|
||||
* In all cases, the file must remain unchanged while the memory map is in use.
|
||||
* In case of error (e.g., file not found, permission denied, etc.), the memory map will be
|
||||
* invalid and view() will return an empty view.
|
||||
* You can check if the memory map is valid by calling is_valid() before using view().
|
||||
*
|
||||
* @param filename the path to the file to memory-map.
|
||||
@@ -4406,8 +4454,14 @@ private:
|
||||
padded_memory_map &operator=(const padded_memory_map &) = delete;
|
||||
const char *data{nullptr};
|
||||
size_t size{0};
|
||||
#ifdef _WIN32
|
||||
// On Windows the underlying storage may either be a memory-mapped view
|
||||
// (released with UnmapViewOfFile) or a heap-allocated padded buffer
|
||||
// (released with delete[]). This flag distinguishes the two cases.
|
||||
bool owns_heap_buffer_{false};
|
||||
#endif
|
||||
};
|
||||
#endif // _WIN32
|
||||
#endif // SIMDJSON_HAS_PADDED_MEMORY_MAP
|
||||
|
||||
|
||||
|
||||
@@ -4700,6 +4754,11 @@ inline padded_string_view pad_with_reserve(std::string& s) noexcept {
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
// On Windows, `padded_memory_map` (when it is enabled) depends on types and
|
||||
// functions declared in <windows.h>. We deliberately do NOT include that
|
||||
// header here: users of simdjson who want `padded_memory_map` on Windows
|
||||
// must include <windows.h> themselves *before* including this header. See
|
||||
// padded_string.h for the detection logic.
|
||||
|
||||
namespace simdjson {
|
||||
namespace internal {
|
||||
@@ -5068,6 +5127,8 @@ inline bool padded_string_builder::reserve(size_t additional) noexcept {
|
||||
}
|
||||
|
||||
|
||||
#if SIMDJSON_HAS_PADDED_MEMORY_MAP
|
||||
|
||||
#ifndef _WIN32
|
||||
simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept {
|
||||
|
||||
@@ -5104,7 +5165,141 @@ simdjson_inline padded_memory_map::~padded_memory_map() noexcept {
|
||||
munmap(const_cast<char *>(data), size + simdjson::SIMDJSON_PADDING);
|
||||
}
|
||||
}
|
||||
#else // _WIN32
|
||||
// Windows 11+ implementation.
|
||||
//
|
||||
// We use the modern Windows memory APIs (CreateFileMapping2 + MapViewOfFile3,
|
||||
// available since Windows 10 1803 and gated on Windows 11 in our build) to
|
||||
// map the file directly into the process address space with zero copies.
|
||||
//
|
||||
// Windows guarantees that after a file view is mapped, any bytes in the
|
||||
// trailing partial page beyond the end of the file are zero-filled. As long
|
||||
// as the file does not end exactly on (or within SIMDJSON_PADDING bytes of)
|
||||
// a page boundary, we therefore get SIMDJSON_PADDING accessible zero bytes
|
||||
// for free at the tail of the view. In the rare edge cases where the tail
|
||||
// is not large enough (about 1.5% of file sizes if sizes were uniformly
|
||||
// distributed), we fall back to reading the file into a heap-allocated
|
||||
// padded buffer. That fallback is still correct — it just performs one
|
||||
// memory copy instead of a zero-copy mapping.
|
||||
simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept {
|
||||
HANDLE file_handle = ::CreateFileA(
|
||||
filename, GENERIC_READ,
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
|
||||
NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
|
||||
if (file_handle == INVALID_HANDLE_VALUE) {
|
||||
return; // file not found or cannot be opened
|
||||
}
|
||||
LARGE_INTEGER file_size_li;
|
||||
if (!::GetFileSizeEx(file_handle, &file_size_li) || file_size_li.QuadPart < 0) {
|
||||
::CloseHandle(file_handle);
|
||||
return; // failed to get file size
|
||||
}
|
||||
#if SIMDJSON_IS_32BITS
|
||||
if (static_cast<unsigned long long>(file_size_li.QuadPart) >
|
||||
static_cast<unsigned long long>(SIZE_MAX - simdjson::SIMDJSON_PADDING)) {
|
||||
::CloseHandle(file_handle);
|
||||
return; // file too large to map on a 32-bit system
|
||||
}
|
||||
#endif
|
||||
size = static_cast<size_t>(file_size_li.QuadPart);
|
||||
|
||||
// Fast zero-copy path: only usable when the last partial page of the file
|
||||
// gives us at least SIMDJSON_PADDING bytes of zero-filled slack.
|
||||
if (size > 0) {
|
||||
SYSTEM_INFO sys_info;
|
||||
::GetSystemInfo(&sys_info);
|
||||
const size_t page_size = static_cast<size_t>(sys_info.dwPageSize);
|
||||
const size_t tail_in_page = size % page_size;
|
||||
const size_t tail_zero_fill = (tail_in_page == 0)
|
||||
? size_t{0}
|
||||
: (page_size - tail_in_page);
|
||||
|
||||
if (tail_zero_fill >= simdjson::SIMDJSON_PADDING) {
|
||||
// Create the section with the new CreateFileMapping2 API.
|
||||
HANDLE mapping = ::CreateFileMapping2(
|
||||
file_handle, /*SecurityAttributes=*/ NULL,
|
||||
/*DesiredAccess=*/ FILE_MAP_READ,
|
||||
/*PageProtection=*/ PAGE_READONLY,
|
||||
/*AllocationAttributes=*/ 0,
|
||||
/*MaximumSize=*/ 0, // 0 => entire file
|
||||
/*Name=*/ NULL,
|
||||
/*ExtendedParameters=*/ NULL, /*ParameterCount=*/ 0);
|
||||
if (mapping != NULL) {
|
||||
// Map the view with the new MapViewOfFile3 API.
|
||||
PVOID view_ptr = ::MapViewOfFile3(
|
||||
mapping, ::GetCurrentProcess(),
|
||||
/*BaseAddress=*/ NULL,
|
||||
/*Offset=*/ 0,
|
||||
/*ViewSize=*/ size,
|
||||
/*AllocationType=*/ 0,
|
||||
/*PageProtection=*/ PAGE_READONLY,
|
||||
/*ExtendedParameters=*/ NULL, /*ParameterCount=*/ 0);
|
||||
::CloseHandle(mapping);
|
||||
if (view_ptr != NULL) {
|
||||
::CloseHandle(file_handle);
|
||||
data = static_cast<const char *>(view_ptr);
|
||||
owns_heap_buffer_ = false;
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Fall through to the buffered-read fallback if the mapping failed.
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback path: the file ends too close to a page boundary (or the
|
||||
// mapping APIs refused) — read the file contents into a heap-allocated
|
||||
// padded buffer. This preserves the class' padding invariant at the cost
|
||||
// of one copy.
|
||||
size_t total_size = size + simdjson::SIMDJSON_PADDING;
|
||||
if (total_size < size) { // overflow guard
|
||||
::CloseHandle(file_handle);
|
||||
size = 0;
|
||||
return;
|
||||
}
|
||||
char *buffer = new (std::nothrow) char[total_size];
|
||||
if (buffer == nullptr) {
|
||||
::CloseHandle(file_handle);
|
||||
size = 0;
|
||||
return;
|
||||
}
|
||||
size_t total_read = 0;
|
||||
while (total_read < size) {
|
||||
size_t remaining = size - total_read;
|
||||
const size_t chunk_limit = static_cast<size_t>(0x40000000UL); // 1 GiB per call
|
||||
DWORD to_read = remaining > chunk_limit
|
||||
? static_cast<DWORD>(chunk_limit)
|
||||
: static_cast<DWORD>(remaining);
|
||||
DWORD bytes_read = 0;
|
||||
if (!::ReadFile(file_handle, buffer + total_read, to_read, &bytes_read, NULL)) {
|
||||
delete[] buffer;
|
||||
::CloseHandle(file_handle);
|
||||
size = 0;
|
||||
return;
|
||||
}
|
||||
if (bytes_read == 0) {
|
||||
// Unexpected EOF: the file shrank while we were reading it.
|
||||
delete[] buffer;
|
||||
::CloseHandle(file_handle);
|
||||
size = 0;
|
||||
return;
|
||||
}
|
||||
total_read += bytes_read;
|
||||
}
|
||||
std::memset(buffer + size, 0, simdjson::SIMDJSON_PADDING);
|
||||
data = buffer;
|
||||
owns_heap_buffer_ = true;
|
||||
::CloseHandle(file_handle);
|
||||
}
|
||||
|
||||
simdjson_inline padded_memory_map::~padded_memory_map() noexcept {
|
||||
if (data == nullptr) { return; }
|
||||
if (owns_heap_buffer_) {
|
||||
delete[] const_cast<char *>(data);
|
||||
} else {
|
||||
::UnmapViewOfFile(data);
|
||||
}
|
||||
}
|
||||
#endif // _WIN32
|
||||
|
||||
simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noexcept simdjson_lifetime_bound {
|
||||
if(!is_valid()) {
|
||||
@@ -5116,7 +5311,8 @@ simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noe
|
||||
simdjson_inline bool padded_memory_map::is_valid() const noexcept {
|
||||
return data != nullptr;
|
||||
}
|
||||
#endif // _WIN32
|
||||
|
||||
#endif // SIMDJSON_HAS_PADDED_MEMORY_MAP
|
||||
|
||||
} // namespace simdjson
|
||||
|
||||
|
||||
+102
-8
@@ -1,13 +1,29 @@
|
||||
// On Windows, padded_memory_map is an opt-in feature gated on the
|
||||
// SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS macro. When that macro is set, the
|
||||
// consumer must also include <windows.h> before <simdjson.h>. We include
|
||||
// the Win32 header here so that -- in configurations that turned the
|
||||
// feature on -- the test actually exercises the Windows path.
|
||||
#ifdef _WIN32
|
||||
#include <cstdlib>
|
||||
// This test is not supported on Windows because it relies on POSIX APIs like
|
||||
// mmap. Please run it on a POSIX-compliant system.
|
||||
int main() { return EXIT_SUCCESS; }
|
||||
#else
|
||||
#ifndef WIN32_LEAN_AND_MEAN
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#endif
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#endif
|
||||
|
||||
#include "simdjson.h"
|
||||
#include "test_macros.h"
|
||||
|
||||
// When SIMDJSON_HAS_PADDED_MEMORY_MAP is 0 (e.g. Windows builds without
|
||||
// SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS, or MinGW configurations that lack
|
||||
// the required SDK gating), compile the test body out and make main()
|
||||
// report success so the test suite still runs as a no-op. This is not a
|
||||
// silent downgrade: users who want the Windows path must explicitly
|
||||
// enable the CMake option `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS`.
|
||||
#if SIMDJSON_HAS_PADDED_MEMORY_MAP
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
bool test_memory_map_exception() {
|
||||
TEST_START();
|
||||
@@ -37,11 +53,89 @@ bool test_memory_map_noexception() {
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
// Verifies that padded_memory_map can feed a streaming parser (iterate_many)
|
||||
// with JSON documents read from a file. This exercises the API that parse_many
|
||||
// / iterate_many users typically want: no extra copy on POSIX, portable fallback
|
||||
// on Windows. The AMAZON_CELLPHONES_NDJSON resource is an NDJSON file so it is
|
||||
// a realistic stress-test for streaming from a memory-mapped file.
|
||||
bool test_memory_map_iterate_many() {
|
||||
TEST_START();
|
||||
simdjson::padded_memory_map map(AMAZON_CELLPHONES_NDJSON);
|
||||
if (!map.is_valid()) {
|
||||
std::cerr << "Failed to memory-map the file " << AMAZON_CELLPHONES_NDJSON << std::endl;
|
||||
return false;
|
||||
}
|
||||
simdjson::padded_string_view view = map.view();
|
||||
simdjson::ondemand::parser parser;
|
||||
simdjson::ondemand::document_stream stream;
|
||||
ASSERT_SUCCESS( parser.iterate_many(view).get(stream) );
|
||||
size_t count = 0;
|
||||
for (auto doc : stream) {
|
||||
ASSERT_SUCCESS( doc.error() );
|
||||
count++;
|
||||
}
|
||||
if (count == 0) {
|
||||
std::cerr << "Expected at least one document in " << AMAZON_CELLPHONES_NDJSON << std::endl;
|
||||
return false;
|
||||
}
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
// Verifies that padded_memory_map also works with the DOM streaming parser
|
||||
// (parse_many). Same rationale as the ondemand variant above.
|
||||
bool test_memory_map_parse_many() {
|
||||
TEST_START();
|
||||
simdjson::padded_memory_map map(AMAZON_CELLPHONES_NDJSON);
|
||||
if (!map.is_valid()) {
|
||||
std::cerr << "Failed to memory-map the file " << AMAZON_CELLPHONES_NDJSON << std::endl;
|
||||
return false;
|
||||
}
|
||||
simdjson::padded_string_view view = map.view();
|
||||
simdjson::dom::parser parser;
|
||||
simdjson::dom::document_stream stream;
|
||||
ASSERT_SUCCESS( parser.parse_many(view).get(stream) );
|
||||
size_t count = 0;
|
||||
for (auto doc : stream) {
|
||||
ASSERT_SUCCESS( doc.error() );
|
||||
count++;
|
||||
}
|
||||
if (count == 0) {
|
||||
std::cerr << "Expected at least one document in " << AMAZON_CELLPHONES_NDJSON << std::endl;
|
||||
return false;
|
||||
}
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
// Ensures that trying to memory-map a file that does not exist leaves the map
|
||||
// in the "invalid" state rather than crashing. This is important on Windows
|
||||
// where the underlying implementation path differs from POSIX.
|
||||
bool test_memory_map_missing_file() {
|
||||
TEST_START();
|
||||
simdjson::padded_memory_map map("this_file_definitely_does_not_exist_123456789.json");
|
||||
if (map.is_valid()) {
|
||||
std::cerr << "Expected is_valid() == false for missing file" << std::endl;
|
||||
return false;
|
||||
}
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
#endif // SIMDJSON_HAS_PADDED_MEMORY_MAP
|
||||
|
||||
int main() {
|
||||
#if SIMDJSON_HAS_PADDED_MEMORY_MAP
|
||||
bool ok = true;
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
return (test_memory_map_exception() && test_memory_map_noexception()) ? EXIT_SUCCESS : EXIT_FAILURE;
|
||||
ok = ok && test_memory_map_exception();
|
||||
#endif
|
||||
ok = ok && test_memory_map_noexception();
|
||||
ok = ok && test_memory_map_iterate_many();
|
||||
ok = ok && test_memory_map_parse_many();
|
||||
ok = ok && test_memory_map_missing_file();
|
||||
return ok ? EXIT_SUCCESS : EXIT_FAILURE;
|
||||
#else
|
||||
return test_memory_map_noexception() ? EXIT_SUCCESS : EXIT_FAILURE;
|
||||
std::cout << "padded_memory_map is disabled in this configuration; "
|
||||
"set SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=ON in CMake to "
|
||||
"enable it on Windows. Test skipped." << std::endl;
|
||||
return EXIT_SUCCESS;
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
Reference in New Issue
Block a user