diff --git a/.github/workflows/vs17-ci.yml b/.github/workflows/vs17-ci.yml index 348e75e74..cca21ba4f 100644 --- a/.github/workflows/vs17-ci.yml +++ b/.github/workflows/vs17-ci.yml @@ -13,18 +13,20 @@ jobs: fail-fast: false matrix: include: - - {gen: Visual Studio 17 2022, arch: Win32, shared: ON, build_type: Release} - - {gen: Visual Studio 17 2022, arch: Win32, shared: OFF, build_type: Release} - - {gen: Visual Studio 17 2022, arch: x64, shared: ON, build_type: Release} - - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Debug} - - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Release} - - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: RelWithDebInfo} + - {gen: Visual Studio 17 2022, arch: Win32, shared: ON, build_type: Release, memory_map: OFF} + - {gen: Visual Studio 17 2022, arch: Win32, shared: OFF, build_type: Release, memory_map: OFF} + - {gen: Visual Studio 17 2022, arch: x64, shared: ON, build_type: Release, memory_map: OFF} + - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Debug, memory_map: OFF} + - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Release, memory_map: OFF} + - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: RelWithDebInfo, memory_map: OFF} + # Exercise the opt-in Windows memory-file mapping path at least once in CI. + - {gen: Visual Studio 17 2022, arch: x64, shared: OFF, build_type: Release, memory_map: ON} steps: - name: checkout uses: actions/checkout@v4 - name: Configure run: | - cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build + cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -DSIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=${{matrix.memory_map}} -B build - name: Build Debug run: cmake --build build --config ${{matrix.build_type}} --verbose - name: Run tests diff --git a/CMakeLists.txt b/CMakeLists.txt index c99bd7205..743bc7e8e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,6 +75,41 @@ if(SIMDJSON_DEVELOPMENT_CHECKS) ) endif() +# padded_memory_map is always available on POSIX. On Windows it is disabled +# by default because it depends on the `CreateFileMapping2` / `MapViewOfFile3` +# APIs, which require Windows 10 version 1803 or later and are exported via +# onecore.lib rather than the default kernel32.lib. Turn this option ON to +# opt into the feature on Windows; simdjson will then set the appropriate +# Windows version macros and link onecore, so everything that links +# simdjson picks up both the compile-time declarations and the import +# library automatically. The option is a no-op on POSIX (where the feature +# is unconditionally enabled). +option(SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS + "Enable simdjson::padded_memory_map on Windows (requires Windows 10 \ +version 1803 or later). Always enabled on POSIX." OFF) +if(SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS) + simdjson_add_props( + target_compile_definitions PUBLIC + SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1 + ) + if(WIN32) + # Raise the Windows version floor so that declares the + # modern memory-mapping APIs, and link the import library that + # actually exports them. _WIN32_WINNT / WINVER / NTDDI_VERSION together + # tell which APIs to light up. + simdjson_add_props( + target_compile_definitions PUBLIC + _WIN32_WINNT=0x0A00 + WINVER=0x0A00 + NTDDI_VERSION=0x0A000006 # NTDDI_WIN10_RS5, Windows 10 version 1809 + ) + simdjson_add_props( + target_link_libraries PUBLIC + onecore + ) + endif() +endif() + if(is_top_project) option(SIMDJSON_INSTALL "Enable target install" ON) option(SIMDJSON_DEVELOPER_MODE "Enable targets for developing simdjson" OFF) diff --git a/doc/basics.md b/doc/basics.md index b081688b3..e961b56f2 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -269,10 +269,35 @@ ondemand::document doc = parser.iterate(simdjson::pad(json)); We recommend against creating many `std::string` or many `std::padded_string` instances in your application to store your JSON data. Consider reusing the same buffers and limiting memory allocations. -**Memory-file mapping (non-Windows).** You can use memory-file mapping to create a `simdjson::padded_string_view` -from a file on disk: +**Memory-file mapping.** You can use `simdjson::padded_memory_map` to create a +`simdjson::padded_string_view` from a file on disk. On POSIX systems (Linux, +macOS, BSD, ...) it uses `mmap` for true zero-copy access and is always +available. On Windows it is an **opt-in** feature because it relies on the +`CreateFileMapping2` / `MapViewOfFile3` APIs (Windows 10, version 1803 or +later) which are exported from `onecore.lib` rather than the default +`kernel32.lib`. To enable it, you must satisfy **all** of the following: + +1. Building simdjson with `-DSIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=ON`, or + defining `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1` and raising + `NTDDI_VERSION` to at least `NTDDI_WIN10_RS4` (Windows 10, version 1803) + and linking `onecore.lib` manually if you are consuming simdjson as a + pre-built library. +2. `#include ` before including simdjson, in every translation + unit that uses `padded_memory_map`. + +The Windows implementation then uses `CreateFileMapping2` / `MapViewOfFile3` +for true zero-copy access whenever possible, with a transparent +buffered-read fallback for files that end too close to a page boundary. + +The availability of the class can be tested with the preprocessor macro +`SIMDJSON_HAS_PADDED_MEMORY_MAP`. ```cpp +#ifdef _WIN32 +#include // Must come BEFORE on Windows +#endif +#include "simdjson.h" +// ... simdjson::padded_memory_map map(myfilename); if (!map.is_valid()) { /* handle error */ } simdjson::padded_string_view view = map.view(); @@ -305,7 +330,7 @@ Some users may want to browse code along with the compiled assembly: | `simdjson::pad(std::string&)` | Adds padding if needed | Returns `padded_string_view` pointing to the (possibly resized) string | References original string | Recommended to silence sanitizers when using `std::string`. | | `padded_string(data, length)` or `padded_string(std::string)` | Automatic (copies into padded buffer) | Explicit copy into owned padded buffer | Owned by `padded_string` | Safe when you want full ownership and padding guaranteed. | | `padded_string_view` (manual) | User guarantees `SIMDJSON_PADDING` extra bytes after the viewed length | User provides pointer + length + capacity | Non-owning view | Low-level; requires careful buffer management. | -| Memory-mapped file (`padded_memory_map`) | Automatic via mapping (non-Windows only) | Creates view with sufficient padding | Non-owning (tied to map lifetime) | Advanced; efficient for large files on Linux/macOS/etc. | +| Memory-mapped file (`padded_memory_map`) | Automatic via mapping / padded read | Creates view with sufficient padding | Non-owning (tied to map lifetime) | Always available on POSIX (zero-copy `mmap`). On Windows, opt-in via `-DSIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=ON` (requires Windows 10 1803+ and links `onecore.lib`) and `#include ` before simdjson; uses `CreateFileMapping2` + `MapViewOfFile3`. | Documents are iterators diff --git a/doc/dom.md b/doc/dom.md index e42c46baa..ee4460c16 100644 --- a/doc/dom.md +++ b/doc/dom.md @@ -127,11 +127,31 @@ codepage, and they may call SetFileApisToOEM accordingly. **Advanced feature:** -On non-Windows systems, you can use memory-file mapping to create a `simdjson::padded_string_view` -from a file on disk. +You can use `simdjson::padded_memory_map` to create a `simdjson::padded_string_view` +from a file on disk without copying the file contents into your own buffer. +On POSIX systems (Linux, macOS, BSD, ...) it uses `mmap` for true zero-copy +access. On Windows it is available as an **opt-in** feature and requires: + +1. Building simdjson with `-DSIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=ON`, or + defining `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1` and raising + `NTDDI_VERSION` to at least `NTDDI_WIN10_RS4` (Windows 10, version 1803) + and linking `onecore.lib` manually if you are consuming simdjson as a + pre-built library. +2. `#include ` before `#include "simdjson.h"` in every + translation unit where you want to use `padded_memory_map`. + +When enabled on Windows, the implementation uses `CreateFileMapping2` and +`MapViewOfFile3` for true zero-copy mapping whenever the file does not end +within `SIMDJSON_PADDING` bytes of a page boundary; otherwise it falls back +to reading the file into a padded heap buffer. If those requirements are +not met, the class is not declared and the code below will fail to compile. ```cpp -// if the macro _WIN32 is defined, this will not work since we do not support Windows +#ifdef _WIN32 +#include // Must come BEFORE on Windows +#endif +#include "simdjson.h" +// ... simdjson::padded_memory_map map(TWITTER_JSON); if (!map.is_valid()) { /* handle error */ } simdjson::padded_string_view view = map.view(); // view is usable while padded_memory_map is in scope diff --git a/doc/iterate_many.md b/doc/iterate_many.md index 2155c5e4c..723686c14 100644 --- a/doc/iterate_many.md +++ b/doc/iterate_many.md @@ -22,6 +22,7 @@ Contents - [Threads](#threads) - [Support](#support) - [API](#api) +- [Streaming directly from a memory-mapped file](#streaming-directly-from-a-memory-mapped-file) - [Use cases](#use-cases) - [Tracking your position](#tracking-your-position) - [Incomplete streams](#incomplete-streams) @@ -156,13 +157,79 @@ for (auto doc : docs) { See [basics.md](basics.md#newline-delimited-json-ndjson-and-json-lines) for an overview of the API. -**Advanced feature:** -On non-Windows systems, you can use memory-file mapping to create a `simdjson::padded_string_view` -from a file on disk. +Streaming directly from a memory-mapped file +-------------------------------------------- + +When your input is a large NDJSON / JSON-lines file on disk, the most efficient +way to feed `iterate_many` is to use `simdjson::padded_memory_map`. It returns +a `padded_string_view` with the right amount of trailing padding, so you can +hand it straight to `iterate_many` without ever copying the file contents into +your own buffer. + +`padded_memory_map` is available on POSIX systems (Linux, macOS, BSD, ...) by +default. On Windows it is an **opt-in** feature with the following +requirements: + +1. Build simdjson with `-DSIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=ON`, or — if + you consume simdjson as a pre-built library — define + `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1`, raise `NTDDI_VERSION` to at + least `NTDDI_WIN10_RS4` (`0x0A000005`, Windows 10 version 1803), and + add `onecore.lib` to your link line yourself. The Windows + implementation uses the modern memory APIs `CreateFileMapping2` / + `MapViewOfFile3`, which are available starting with that version of + Windows and are exported by `onecore.lib`. +2. `#include ` before `#include "simdjson.h"` in every + translation unit where you want to use `padded_memory_map`. simdjson + deliberately does not pull in `` itself, so the class is + only declared when the Win32 types are already visible. + +If either requirement is not met on Windows, the `padded_memory_map` class is +not declared at all and any code that references it fails to compile with an +"unknown identifier" error. The availability of the class can be tested with +the macro `SIMDJSON_HAS_PADDED_MEMORY_MAP`. + +On POSIX, `padded_memory_map` uses `mmap` to map the file directly into +memory with zero copies. On Windows (when enabled), it uses +`CreateFileMapping2` + `MapViewOfFile3` for true zero-copy mapping +whenever the file does not end within `SIMDJSON_PADDING` bytes of a page +boundary; for those rare cases, it transparently falls back to reading +the file into a heap-allocated padded buffer so that the returned view +always has `SIMDJSON_PADDING` accessible zero bytes after the file content. + +```cpp +#ifdef _WIN32 +#include // Must come BEFORE on Windows +#endif +#include "simdjson.h" + +// ... + +simdjson::padded_memory_map map("huge_stream.ndjson"); +if (!map.is_valid()) { /* file missing, unreadable, too large, ... */ return; } + +simdjson::ondemand::parser parser; +simdjson::ondemand::document_stream stream; +auto error = parser.iterate_many(map.view()).get(stream); +if (error) { std::cerr << error << std::endl; return; } + +for (auto doc : stream) { + // process each JSON document in the stream + std::cout << doc << std::endl; +} +``` + +Important lifetime rule: the `padded_string_view` returned by `map.view()` is +only valid while the `padded_memory_map` instance is alive, so keep `map` +alive for as long as you are iterating the stream. + +The file must not be modified while the memory map is in use. If you need a +fully independent copy of the data, use `simdjson::padded_string::load(...)` +instead. + +If you prefer single-document parsing on a memory-mapped file, the same +pattern applies to `parser.iterate(...)`: ```cpp -// If the macro _WIN32 is defined, this will not work since we do not support memory-file mapping -// under Windows at this time. simdjson::padded_memory_map map(myfilename); if (!map.is_valid()) { /* handle error */ } simdjson::padded_string_view view = map.view(); // view is usable while padded_memory_map is in scope diff --git a/doc/parse_many.md b/doc/parse_many.md index 88fe10b23..99762346e 100644 --- a/doc/parse_many.md +++ b/doc/parse_many.md @@ -18,6 +18,7 @@ Contents - [How it works](#how-it-works) - [Support](#support) - [API](#api) +- [Streaming directly from a memory-mapped file](#streaming-directly-from-a-memory-mapped-file) - [Use cases](#use-cases) - [Tracking your position](#tracking-your-position) - [Incomplete streams](#incomplete-streams) @@ -218,17 +219,83 @@ got full document at 29 -**Advanced feature:** -On non-Windows systems, you can use memory-file mapping to create a `simdjson::padded_string_view` -from a file on disk. +Streaming directly from a memory-mapped file +-------------------------------------------- + +When your input is a large NDJSON / JSON-lines file on disk, the most +efficient way to feed `parse_many` is to use `simdjson::padded_memory_map`. +It returns a `padded_string_view` with the right amount of trailing padding, +so you can pass it directly to `parse_many` without copying the file content +into your own buffer first. + +`padded_memory_map` is available on POSIX systems (Linux, macOS, BSD, ...) by +default. On Windows it is an **opt-in** feature with the following +requirements: + +1. Build simdjson with `-DSIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=ON`, or — if + you consume simdjson as a pre-built library — define + `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1`, raise `NTDDI_VERSION` to at + least `NTDDI_WIN10_RS4` (`0x0A000005`, Windows 10 version 1803), and + add `onecore.lib` to your link line yourself. The Windows + implementation uses the modern memory APIs `CreateFileMapping2` / + `MapViewOfFile3`, which are available starting with that version of + Windows and are exported by `onecore.lib`. +2. `#include ` before `#include "simdjson.h"` in every + translation unit where you want to use `padded_memory_map`. simdjson + deliberately does not pull in `` itself, so the class is + only declared when the Win32 types are already visible. + +If either requirement is not met on Windows, the `padded_memory_map` class is +not declared at all and any code that references it fails to compile with an +"unknown identifier" error. The availability of the class can be tested with +the macro `SIMDJSON_HAS_PADDED_MEMORY_MAP`. + +On POSIX, `padded_memory_map` uses `mmap` to map the file directly into +memory with zero copies. On Windows (when enabled), it uses +`CreateFileMapping2` + `MapViewOfFile3` for true zero-copy mapping +whenever the file does not end within `SIMDJSON_PADDING` bytes of a page +boundary; for those rare cases, it transparently falls back to reading +the file into a heap-allocated padded buffer so that the returned view +always has `SIMDJSON_PADDING` accessible zero bytes after the file content. + +```cpp +#ifdef _WIN32 +#include // Must come BEFORE on Windows +#endif +#include "simdjson.h" + +// ... + +simdjson::padded_memory_map map("huge_stream.ndjson"); +if (!map.is_valid()) { /* file missing, unreadable, too large, ... */ return; } + +simdjson::dom::parser parser; +simdjson::dom::document_stream stream; +auto error = parser.parse_many(map.view()).get(stream); +if (error) { std::cerr << error << std::endl; return; } + +for (auto doc : stream) { + // process each JSON document in the stream + std::cout << doc << std::endl; +} +``` + +Important lifetime rule: the `padded_string_view` returned by `map.view()` is +only valid while the `padded_memory_map` instance is alive, so keep `map` +alive for as long as you are iterating the stream. + +The file must not be modified while the memory map is in use. If you need a +fully independent copy of the data, use `simdjson::padded_string::load(...)` +instead. + +If you prefer single-document parsing on a memory-mapped file, the same +pattern applies to `parser.parse(...)`: ```cpp -// If the macro _WIN32 is defined, this will not work since we do not support memory-file mapping -// under Windows at this time. simdjson::padded_memory_map map(myfilename); if (!map.is_valid()) { /* handle error */ } simdjson::padded_string_view view = map.view(); // view is usable while padded_memory_map is in scope -ondemand::document doc = parser.iterate(view); // parse the JSON +simdjson::dom::element doc = parser.parse(view); // parse the JSON ``` Incomplete streams diff --git a/include/simdjson/dom/parser-inl.h b/include/simdjson/dom/parser-inl.h index 141c3db84..941909f82 100644 --- a/include/simdjson/dom/parser-inl.h +++ b/include/simdjson/dom/parser-inl.h @@ -181,6 +181,9 @@ inline simdjson_result parser::parse_many(const std::string &s, inline simdjson_result parser::parse_many(const padded_string &s, size_t batch_size) noexcept { return parse_many(s.data(), s.length(), batch_size); } +inline simdjson_result parser::parse_many(const padded_string_view &v, size_t batch_size) noexcept { + return parse_many(v.data(), v.length(), batch_size); +} inline simdjson_result parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) noexcept { if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } @@ -217,6 +220,9 @@ inline simdjson_result parser::parse_many(const std::string &s, inline simdjson_result parser::parse_many(const padded_string &s, size_t batch_size, stream_format format) noexcept { return parse_many(s.data(), s.length(), batch_size, format); } +inline simdjson_result parser::parse_many(const padded_string_view &v, size_t batch_size, stream_format format) noexcept { + return parse_many(v.data(), v.length(), batch_size, format); +} simdjson_inline size_t parser::capacity() const noexcept { return implementation ? implementation->capacity() : 0; diff --git a/include/simdjson/dom/parser.h b/include/simdjson/dom/parser.h index dd1f7bd1e..c731d2814 100644 --- a/include/simdjson/dom/parser.h +++ b/include/simdjson/dom/parser.h @@ -490,6 +490,16 @@ public: /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result parse_many(const padded_string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; inline simdjson_result parse_many(const padded_string &&s, size_t batch_size) = delete;// unsafe + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) + * + * Because padded_string_view guarantees SIMDJSON_PADDING trailing bytes, this + * overload is safe to use with buffers that the caller owns elsewhere (for + * example, a padded_memory_map), with no extra copy. Without this overload, + * passing a padded_string_view would silently bind to the padded_string + * overload via an implicit conversion, allocating and copying the input, and + * — because that temporary is destroyed at the end of the full-expression — + * leaving the returned document_stream pointing at freed memory. */ + inline simdjson_result parse_many(const padded_string_view &v, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; /** @private We do not want to allow implicit conversion from C string to std::string. */ simdjson_result parse_many(const char *buf, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept = delete; @@ -510,6 +520,8 @@ public: inline simdjson_result parse_many(const std::string &s, size_t batch_size, stream_format format) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */ inline simdjson_result parse_many(const padded_string &s, size_t batch_size, stream_format format) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */ + inline simdjson_result parse_many(const padded_string_view &v, size_t batch_size, stream_format format) noexcept; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length diff --git a/include/simdjson/padded_string-inl.h b/include/simdjson/padded_string-inl.h index 63344985f..7040ab354 100644 --- a/include/simdjson/padded_string-inl.h +++ b/include/simdjson/padded_string-inl.h @@ -10,13 +10,18 @@ #include #include -#ifndef _WIN32 +#if SIMDJSON_HAS_UNISTD_H #include #include #include #include #include #endif +// On Windows, `padded_memory_map` (when it is enabled) depends on types and +// functions declared in . We deliberately do NOT include that +// header here: users of simdjson who want `padded_memory_map` on Windows +// must include themselves *before* including this header. See +// padded_string.h for the detection logic. namespace simdjson { namespace internal { @@ -385,7 +390,9 @@ inline bool padded_string_builder::reserve(size_t additional) noexcept { } -#ifndef _WIN32 +#if SIMDJSON_HAS_PADDED_MEMORY_MAP + +#if SIMDJSON_HAS_UNISTD_H simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept { int fd = open(filename, O_RDONLY); @@ -421,7 +428,132 @@ simdjson_inline padded_memory_map::~padded_memory_map() noexcept { munmap(const_cast(data), size + simdjson::SIMDJSON_PADDING); } } +#elif defined(_WIN32) +// Windows zero-copy implementation using placeholder virtual memory. +// +// We use the modern Windows memory APIs (VirtualAlloc2, CreateFileMapping2, +// MapViewOfFile3 — available since Windows 10 1803) to map the file into a +// contiguous virtual address range that includes at least SIMDJSON_PADDING +// zero bytes after the file content, with no data copies. +// +// Strategy: +// 1. If rounding the file size up to the allocation granularity already +// exceeds file_size + SIMDJSON_PADDING, the OS page zero-fill provides +// the padding and we use a simple MapViewOfFile3 call. +// 2. Otherwise we reserve a contiguous placeholder region via VirtualAlloc2, +// split it at the granularity-aligned file boundary, map the file into +// the first part, and commit zero pages for the second part (padding). +simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept { + HANDLE file_handle = ::CreateFileA( + filename, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) { + return; + } + LARGE_INTEGER file_size_li; + if (!::GetFileSizeEx(file_handle, &file_size_li) || file_size_li.QuadPart < 0) { + ::CloseHandle(file_handle); + return; + } +#if SIMDJSON_IS_32BITS + if (static_cast(file_size_li.QuadPart) > + static_cast(SIZE_MAX - simdjson::SIMDJSON_PADDING)) { + ::CloseHandle(file_handle); + return; + } +#endif + size = static_cast(file_size_li.QuadPart); + if (size == 0) { + ::CloseHandle(file_handle); + return; + } + HANDLE section = ::CreateFileMapping2( + file_handle, NULL, FILE_MAP_READ, PAGE_READONLY, + 0, 0, NULL, NULL, 0); + ::CloseHandle(file_handle); + if (section == NULL) { + return; + } + + SYSTEM_INFO si; + ::GetSystemInfo(&si); + const size_t granularity = static_cast(si.dwAllocationGranularity); + const size_t file_region = (size + granularity - 1) & ~(granularity - 1); + const size_t total_needed = size + simdjson::SIMDJSON_PADDING; + + if (file_region >= total_needed) { + // The zero-fill in the last page already covers the padding. + PVOID view = ::MapViewOfFile3( + section, ::GetCurrentProcess(), NULL, 0, 0, + 0, PAGE_READONLY, NULL, 0); + ::CloseHandle(section); + if (view != NULL) { + data = static_cast(view); + } + return; + } + + // We need extra zero pages beyond the file region. Use the placeholder API + // to get a contiguous virtual address range spanning both the file mapping + // and the zero-filled padding. + const size_t padding_region = + ((total_needed - file_region) + granularity - 1) & ~(granularity - 1); + const size_t reserve_size = file_region + padding_region; + + // Reserve a contiguous placeholder. + PVOID placeholder = ::VirtualAlloc2( + ::GetCurrentProcess(), NULL, reserve_size, + MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, NULL, 0); + if (placeholder == NULL) { + ::CloseHandle(section); + return; + } + + // Split into two placeholders at the file_region boundary. + if (!::VirtualFree(placeholder, file_region, + MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER)) { + ::VirtualFree(placeholder, 0, MEM_RELEASE); + ::CloseHandle(section); + return; + } + + // Map the file into the first placeholder. + PVOID file_view = ::MapViewOfFile3( + section, ::GetCurrentProcess(), placeholder, 0, file_region, + MEM_REPLACE_PLACEHOLDER, PAGE_READONLY, NULL, 0); + ::CloseHandle(section); + if (file_view == NULL) { + ::VirtualFree(placeholder, 0, MEM_RELEASE); + ::VirtualFree(static_cast(placeholder) + file_region, + 0, MEM_RELEASE); + return; + } + + // Commit zero pages in the second placeholder (the padding). + void *pad = static_cast(placeholder) + file_region; + PVOID padding_ptr = ::VirtualAlloc2( + ::GetCurrentProcess(), pad, padding_region, + MEM_REPLACE_PLACEHOLDER | MEM_COMMIT, PAGE_READONLY, NULL, 0); + if (padding_ptr == NULL) { + ::UnmapViewOfFile(file_view); + ::VirtualFree(pad, 0, MEM_RELEASE); + return; + } + + data = static_cast(file_view); + padding_view_ = padding_ptr; +} + +simdjson_inline padded_memory_map::~padded_memory_map() noexcept { + if (data == nullptr) { return; } + ::UnmapViewOfFile(data); + if (padding_view_ != nullptr) { + ::VirtualFree(padding_view_, 0, MEM_RELEASE); + } +} +#endif // POSIX or _WIN32 simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noexcept simdjson_lifetime_bound { if(!is_valid()) { @@ -433,7 +565,8 @@ simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noe simdjson_inline bool padded_memory_map::is_valid() const noexcept { return data != nullptr; } -#endif // _WIN32 + +#endif // SIMDJSON_HAS_PADDED_MEMORY_MAP } // namespace simdjson diff --git a/include/simdjson/padded_string.h b/include/simdjson/padded_string.h index 1e83e5f62..a32d4a746 100644 --- a/include/simdjson/padded_string.h +++ b/include/simdjson/padded_string.h @@ -277,11 +277,26 @@ inline std::ostream& operator<<(std::ostream& out, const padded_string& s) { ret inline std::ostream& operator<<(std::ostream& out, simdjson_result &s) noexcept(false) { return out << s.value(); } #endif - -#ifndef _WIN32 +#if SIMDJSON_HAS_PADDED_MEMORY_MAP /** * A class representing a memory-mapped file with padding. - * It is only available on non-Windows platforms, as Windows has different APIs for memory mapping. + * + * On POSIX systems (Linux, macOS, BSD, ...), this uses `mmap` to map the file + * contents directly into memory, which is efficient for large files (no copy). + * + * On Windows, this class is disabled by default and must be opted into at + * build time by defining `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1`. When + * enabled, `` must also be included before `` and + * the compilation must target Windows 10, version 1803 or later. The + * Windows implementation uses the modern memory APIs (`VirtualAlloc2`, + * `CreateFileMapping2`, `MapViewOfFile3`) with the placeholder virtual + * memory mechanism to always achieve true zero-copy mapping with + * contiguous zero-filled padding. + * + * Either way, the resulting `padded_string_view` carries at least + * `SIMDJSON_PADDING` bytes of accessible zero-filled padding after the file + * content, so it can be consumed directly by the simdjson parsers (including + * `parse_many` / `iterate_many`). */ class padded_memory_map { public: @@ -289,9 +304,11 @@ public: * Create a new padded memory map for the given file. * After creating the memory map, you can call view() to get a padded_string_view of the file content. * The memory map will be automatically released when the padded_memory_map instance is destroyed. - * Note that the file content is not copied, so this is efficient for large files. However, - * the file must remain unchanged while the memory map is in use. In case of error (e.g., file not found, - * permission denied, etc.), the memory map will be invalid and view() will return an empty view. + * On POSIX systems, the file content is not copied, so this is efficient for large files. + * On Windows, the file is mapped into memory via `MapViewOfFile3` (zero-copy). + * In all cases, the file must remain unchanged while the memory map is in use. + * In case of error (e.g., file not found, permission denied, etc.), the memory map will be + * invalid and view() will return an empty view. * You can check if the memory map is valid by calling is_valid() before using view(). * * @param filename the path to the file to memory-map. @@ -328,8 +345,14 @@ private: padded_memory_map &operator=(const padded_memory_map &) = delete; const char *data{nullptr}; size_t size{0}; +#ifdef _WIN32 + // When the file ends near an allocation-granularity boundary, we use the + // placeholder API to append zero-filled padding pages. This pointer tracks + // that region so the destructor can release it with VirtualFree. + void *padding_view_{nullptr}; +#endif }; -#endif // _WIN32 +#endif // SIMDJSON_HAS_PADDED_MEMORY_MAP diff --git a/include/simdjson/padded_string_view-inl.h b/include/simdjson/padded_string_view-inl.h index 3d515c269..c6f589dc7 100644 --- a/include/simdjson/padded_string_view-inl.h +++ b/include/simdjson/padded_string_view-inl.h @@ -7,7 +7,7 @@ #include /* memcmp */ // for page size computation. -#if defined(__unix__) || defined(__APPLE__) || defined(__linux__) +#if SIMDJSON_HAS_UNISTD_H #include #if defined(__APPLE__) #include @@ -113,7 +113,7 @@ inline uint32_t get_page_size() noexcept { return static_cast(si.dwPageSize); }(); return cached; -#elif defined(__unix__) || defined(__APPLE__) || defined(__linux__) +#elif SIMDJSON_HAS_UNISTD_H static const uint32_t cached = []() -> uint32_t { long page_size = sysconf(_SC_PAGESIZE); if (page_size > 0) { diff --git a/include/simdjson/portability.h b/include/simdjson/portability.h index bff2163ba..aaf3692cb 100644 --- a/include/simdjson/portability.h +++ b/include/simdjson/portability.h @@ -285,5 +285,53 @@ using std::size_t; #endif #endif +#ifndef SIMDJSON_HAS_UNISTD_H +#if defined(__unix__) || defined(__APPLE__) || defined(__linux__) +#define SIMDJSON_HAS_UNISTD_H 1 +#else +#define SIMDJSON_HAS_UNISTD_H 0 +#endif +#endif + +// padded_memory_map availability. +// +// On POSIX platforms the class is always available: the implementation uses +// `mmap` (and a trailing anonymous page for padding) from . +// +// On Windows the class is disabled by default and must be explicitly +// opted into by defining `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1`. Enabling +// it requires: +// 1. `` has been included *before* `` (so that +// this header can see the Win32 types and the `_WINDOWS_` include +// guard), +// 2. the compilation targets Windows 10, version 1803 or later +// (i.e. `NTDDI_VERSION >= NTDDI_WIN10_RS4`, `0x0A000005`). This is +// required because the implementation relies on the modern memory +// APIs introduced with that version (`CreateFileMapping2` / +// `MapViewOfFile3`), +// 3. the link step pulls in an import library that exports those APIs, +// typically `onecore.lib` (or `mincore.lib`). +// +// The `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS` CMake option arranges (1)-(3) +// automatically when building simdjson with its own CMake. Consumers using +// simdjson as a pre-built library are responsible for setting the macro, +// the Windows version macros, and the link library themselves. +// +// If the opt-in conditions are not met on Windows, `padded_memory_map` +// simply does not exist — any attempt to use it fails at compile time +// with an "unknown identifier" diagnostic rather than silently degrading. +// +// The SIMDJSON_HAS_PADDED_MEMORY_MAP macro reflects whether the class is +// available in the current translation unit. Users may test this macro to +// conditionally compile code that depends on padded_memory_map. +#ifndef SIMDJSON_HAS_PADDED_MEMORY_MAP + #if defined(__unix__) || defined(__APPLE__) || defined(__linux__) + #define SIMDJSON_HAS_PADDED_MEMORY_MAP 1 + #elif defined(_WINDOWS_) && defined(SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS) && SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS + #define SIMDJSON_HAS_PADDED_MEMORY_MAP 1 + #else + #define SIMDJSON_HAS_PADDED_MEMORY_MAP 0 + #endif +#endif #endif // SIMDJSON_PORTABILITY_H diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index 7a5d7f17f..b5eb6c87b 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -4356,10 +4356,55 @@ inline std::ostream& operator<<(std::ostream& out, simdjson_result. +// +// On Windows the class is only available when all of the following hold: +// 1. has been included *before* (so that this +// header can see the Win32 types and the `_WINDOWS_` include guard), +// 2. the compilation targets Windows 11 or later (NTDDI_VERSION +// >= NTDDI_WIN10_CO, 0x0A00000B). This is required because the +// implementation relies on the modern memory APIs introduced with +// that version (CreateFileMapping2 / MapViewOfFile3). +// +// If those conditions are not met on Windows, `padded_memory_map` simply +// does not exist — any attempt to use it fails at compile time with an +// "unknown identifier" diagnostic rather than silently degrading. +// +// The SIMDJSON_HAS_PADDED_MEMORY_MAP macro reflects whether the class is +// available in the current translation unit. Users may test this macro to +// conditionally compile code that depends on padded_memory_map. +#ifndef SIMDJSON_HAS_PADDED_MEMORY_MAP + #if !defined(_WIN32) + #define SIMDJSON_HAS_PADDED_MEMORY_MAP 1 + #elif defined(_WINDOWS_) && defined(NTDDI_VERSION) && (NTDDI_VERSION >= 0x0A00000B /* NTDDI_WIN10_CO — Windows 11 */) + #define SIMDJSON_HAS_PADDED_MEMORY_MAP 1 + #else + #define SIMDJSON_HAS_PADDED_MEMORY_MAP 0 + #endif +#endif + +#if SIMDJSON_HAS_PADDED_MEMORY_MAP /** * A class representing a memory-mapped file with padding. - * It is only available on non-Windows platforms, as Windows has different APIs for memory mapping. + * + * On POSIX systems (Linux, macOS, BSD, ...), this uses `mmap` to map the file + * contents directly into memory, which is efficient for large files (no copy). + * + * On Windows, this class is only available when `` is included + * before `` and the compilation targets Windows 11 or later + * (NTDDI_VERSION >= NTDDI_WIN10_CO). The Windows implementation uses the + * modern memory APIs (`CreateFileMapping2` / `MapViewOfFile3`) to map the + * file with true zero-copy semantics whenever the last page of the file + * provides enough trailing zero-fill for SIMDJSON_PADDING bytes; otherwise + * it falls back to a heap-allocated padded buffer populated with `ReadFile`. + * + * Either way, the resulting `padded_string_view` carries at least + * `SIMDJSON_PADDING` bytes of accessible zero-filled padding after the file + * content, so it can be consumed directly by the simdjson parsers (including + * `parse_many` / `iterate_many`). */ class padded_memory_map { public: @@ -4367,9 +4412,12 @@ public: * Create a new padded memory map for the given file. * After creating the memory map, you can call view() to get a padded_string_view of the file content. * The memory map will be automatically released when the padded_memory_map instance is destroyed. - * Note that the file content is not copied, so this is efficient for large files. However, - * the file must remain unchanged while the memory map is in use. In case of error (e.g., file not found, - * permission denied, etc.), the memory map will be invalid and view() will return an empty view. + * On POSIX systems, the file content is not copied, so this is efficient for large files. + * On Windows, the file is mapped into memory via `MapViewOfFile3` whenever possible + * (zero-copy) and otherwise read into a heap-allocated padded buffer. + * In all cases, the file must remain unchanged while the memory map is in use. + * In case of error (e.g., file not found, permission denied, etc.), the memory map will be + * invalid and view() will return an empty view. * You can check if the memory map is valid by calling is_valid() before using view(). * * @param filename the path to the file to memory-map. @@ -4406,8 +4454,14 @@ private: padded_memory_map &operator=(const padded_memory_map &) = delete; const char *data{nullptr}; size_t size{0}; +#ifdef _WIN32 + // On Windows the underlying storage may either be a memory-mapped view + // (released with UnmapViewOfFile) or a heap-allocated padded buffer + // (released with delete[]). This flag distinguishes the two cases. + bool owns_heap_buffer_{false}; +#endif }; -#endif // _WIN32 +#endif // SIMDJSON_HAS_PADDED_MEMORY_MAP @@ -4700,6 +4754,11 @@ inline padded_string_view pad_with_reserve(std::string& s) noexcept { #include #include #endif +// On Windows, `padded_memory_map` (when it is enabled) depends on types and +// functions declared in . We deliberately do NOT include that +// header here: users of simdjson who want `padded_memory_map` on Windows +// must include themselves *before* including this header. See +// padded_string.h for the detection logic. namespace simdjson { namespace internal { @@ -5068,6 +5127,8 @@ inline bool padded_string_builder::reserve(size_t additional) noexcept { } +#if SIMDJSON_HAS_PADDED_MEMORY_MAP + #ifndef _WIN32 simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept { @@ -5104,7 +5165,141 @@ simdjson_inline padded_memory_map::~padded_memory_map() noexcept { munmap(const_cast(data), size + simdjson::SIMDJSON_PADDING); } } +#else // _WIN32 +// Windows 11+ implementation. +// +// We use the modern Windows memory APIs (CreateFileMapping2 + MapViewOfFile3, +// available since Windows 10 1803 and gated on Windows 11 in our build) to +// map the file directly into the process address space with zero copies. +// +// Windows guarantees that after a file view is mapped, any bytes in the +// trailing partial page beyond the end of the file are zero-filled. As long +// as the file does not end exactly on (or within SIMDJSON_PADDING bytes of) +// a page boundary, we therefore get SIMDJSON_PADDING accessible zero bytes +// for free at the tail of the view. In the rare edge cases where the tail +// is not large enough (about 1.5% of file sizes if sizes were uniformly +// distributed), we fall back to reading the file into a heap-allocated +// padded buffer. That fallback is still correct — it just performs one +// memory copy instead of a zero-copy mapping. +simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept { + HANDLE file_handle = ::CreateFileA( + filename, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) { + return; // file not found or cannot be opened + } + LARGE_INTEGER file_size_li; + if (!::GetFileSizeEx(file_handle, &file_size_li) || file_size_li.QuadPart < 0) { + ::CloseHandle(file_handle); + return; // failed to get file size + } +#if SIMDJSON_IS_32BITS + if (static_cast(file_size_li.QuadPart) > + static_cast(SIZE_MAX - simdjson::SIMDJSON_PADDING)) { + ::CloseHandle(file_handle); + return; // file too large to map on a 32-bit system + } +#endif + size = static_cast(file_size_li.QuadPart); + // Fast zero-copy path: only usable when the last partial page of the file + // gives us at least SIMDJSON_PADDING bytes of zero-filled slack. + if (size > 0) { + SYSTEM_INFO sys_info; + ::GetSystemInfo(&sys_info); + const size_t page_size = static_cast(sys_info.dwPageSize); + const size_t tail_in_page = size % page_size; + const size_t tail_zero_fill = (tail_in_page == 0) + ? size_t{0} + : (page_size - tail_in_page); + + if (tail_zero_fill >= simdjson::SIMDJSON_PADDING) { + // Create the section with the new CreateFileMapping2 API. + HANDLE mapping = ::CreateFileMapping2( + file_handle, /*SecurityAttributes=*/ NULL, + /*DesiredAccess=*/ FILE_MAP_READ, + /*PageProtection=*/ PAGE_READONLY, + /*AllocationAttributes=*/ 0, + /*MaximumSize=*/ 0, // 0 => entire file + /*Name=*/ NULL, + /*ExtendedParameters=*/ NULL, /*ParameterCount=*/ 0); + if (mapping != NULL) { + // Map the view with the new MapViewOfFile3 API. + PVOID view_ptr = ::MapViewOfFile3( + mapping, ::GetCurrentProcess(), + /*BaseAddress=*/ NULL, + /*Offset=*/ 0, + /*ViewSize=*/ size, + /*AllocationType=*/ 0, + /*PageProtection=*/ PAGE_READONLY, + /*ExtendedParameters=*/ NULL, /*ParameterCount=*/ 0); + ::CloseHandle(mapping); + if (view_ptr != NULL) { + ::CloseHandle(file_handle); + data = static_cast(view_ptr); + owns_heap_buffer_ = false; + return; + } + } + // Fall through to the buffered-read fallback if the mapping failed. + } + } + + // Fallback path: the file ends too close to a page boundary (or the + // mapping APIs refused) — read the file contents into a heap-allocated + // padded buffer. This preserves the class' padding invariant at the cost + // of one copy. + size_t total_size = size + simdjson::SIMDJSON_PADDING; + if (total_size < size) { // overflow guard + ::CloseHandle(file_handle); + size = 0; + return; + } + char *buffer = new (std::nothrow) char[total_size]; + if (buffer == nullptr) { + ::CloseHandle(file_handle); + size = 0; + return; + } + size_t total_read = 0; + while (total_read < size) { + size_t remaining = size - total_read; + const size_t chunk_limit = static_cast(0x40000000UL); // 1 GiB per call + DWORD to_read = remaining > chunk_limit + ? static_cast(chunk_limit) + : static_cast(remaining); + DWORD bytes_read = 0; + if (!::ReadFile(file_handle, buffer + total_read, to_read, &bytes_read, NULL)) { + delete[] buffer; + ::CloseHandle(file_handle); + size = 0; + return; + } + if (bytes_read == 0) { + // Unexpected EOF: the file shrank while we were reading it. + delete[] buffer; + ::CloseHandle(file_handle); + size = 0; + return; + } + total_read += bytes_read; + } + std::memset(buffer + size, 0, simdjson::SIMDJSON_PADDING); + data = buffer; + owns_heap_buffer_ = true; + ::CloseHandle(file_handle); +} + +simdjson_inline padded_memory_map::~padded_memory_map() noexcept { + if (data == nullptr) { return; } + if (owns_heap_buffer_) { + delete[] const_cast(data); + } else { + ::UnmapViewOfFile(data); + } +} +#endif // _WIN32 simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noexcept simdjson_lifetime_bound { if(!is_valid()) { @@ -5116,7 +5311,8 @@ simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noe simdjson_inline bool padded_memory_map::is_valid() const noexcept { return data != nullptr; } -#endif // _WIN32 + +#endif // SIMDJSON_HAS_PADDED_MEMORY_MAP } // namespace simdjson diff --git a/tests/memory_map_tests.cpp b/tests/memory_map_tests.cpp index e6b6adb41..eca814694 100644 --- a/tests/memory_map_tests.cpp +++ b/tests/memory_map_tests.cpp @@ -1,13 +1,29 @@ +// On Windows, padded_memory_map is an opt-in feature gated on the +// SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS macro. When that macro is set, the +// consumer must also include before . We include +// the Win32 header here so that -- in configurations that turned the +// feature on -- the test actually exercises the Windows path. #ifdef _WIN32 -#include -// This test is not supported on Windows because it relies on POSIX APIs like -// mmap. Please run it on a POSIX-compliant system. -int main() { return EXIT_SUCCESS; } -#else +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif #include "simdjson.h" #include "test_macros.h" +// When SIMDJSON_HAS_PADDED_MEMORY_MAP is 0 (e.g. Windows builds without +// SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS, or MinGW configurations that lack +// the required SDK gating), compile the test body out and make main() +// report success so the test suite still runs as a no-op. This is not a +// silent downgrade: users who want the Windows path must explicitly +// enable the CMake option `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS`. +#if SIMDJSON_HAS_PADDED_MEMORY_MAP + #if SIMDJSON_EXCEPTIONS bool test_memory_map_exception() { TEST_START(); @@ -37,11 +53,89 @@ bool test_memory_map_noexception() { TEST_SUCCEED(); } +// Verifies that padded_memory_map can feed a streaming parser (iterate_many) +// with JSON documents read from a file. This exercises the API that parse_many +// / iterate_many users typically want: no extra copy on POSIX, portable fallback +// on Windows. The AMAZON_CELLPHONES_NDJSON resource is an NDJSON file so it is +// a realistic stress-test for streaming from a memory-mapped file. +bool test_memory_map_iterate_many() { + TEST_START(); + simdjson::padded_memory_map map(AMAZON_CELLPHONES_NDJSON); + if (!map.is_valid()) { + std::cerr << "Failed to memory-map the file " << AMAZON_CELLPHONES_NDJSON << std::endl; + return false; + } + simdjson::padded_string_view view = map.view(); + simdjson::ondemand::parser parser; + simdjson::ondemand::document_stream stream; + ASSERT_SUCCESS( parser.iterate_many(view).get(stream) ); + size_t count = 0; + for (auto doc : stream) { + ASSERT_SUCCESS( doc.error() ); + count++; + } + if (count == 0) { + std::cerr << "Expected at least one document in " << AMAZON_CELLPHONES_NDJSON << std::endl; + return false; + } + TEST_SUCCEED(); +} + +// Verifies that padded_memory_map also works with the DOM streaming parser +// (parse_many). Same rationale as the ondemand variant above. +bool test_memory_map_parse_many() { + TEST_START(); + simdjson::padded_memory_map map(AMAZON_CELLPHONES_NDJSON); + if (!map.is_valid()) { + std::cerr << "Failed to memory-map the file " << AMAZON_CELLPHONES_NDJSON << std::endl; + return false; + } + simdjson::padded_string_view view = map.view(); + simdjson::dom::parser parser; + simdjson::dom::document_stream stream; + ASSERT_SUCCESS( parser.parse_many(view).get(stream) ); + size_t count = 0; + for (auto doc : stream) { + ASSERT_SUCCESS( doc.error() ); + count++; + } + if (count == 0) { + std::cerr << "Expected at least one document in " << AMAZON_CELLPHONES_NDJSON << std::endl; + return false; + } + TEST_SUCCEED(); +} + +// Ensures that trying to memory-map a file that does not exist leaves the map +// in the "invalid" state rather than crashing. This is important on Windows +// where the underlying implementation path differs from POSIX. +bool test_memory_map_missing_file() { + TEST_START(); + simdjson::padded_memory_map map("this_file_definitely_does_not_exist_123456789.json"); + if (map.is_valid()) { + std::cerr << "Expected is_valid() == false for missing file" << std::endl; + return false; + } + TEST_SUCCEED(); +} + +#endif // SIMDJSON_HAS_PADDED_MEMORY_MAP + int main() { +#if SIMDJSON_HAS_PADDED_MEMORY_MAP + bool ok = true; #if SIMDJSON_EXCEPTIONS - return (test_memory_map_exception() && test_memory_map_noexception()) ? EXIT_SUCCESS : EXIT_FAILURE; + ok = ok && test_memory_map_exception(); +#endif + ok = ok && test_memory_map_noexception(); + ok = ok && test_memory_map_iterate_many(); + ok = ok && test_memory_map_parse_many(); + ok = ok && test_memory_map_missing_file(); + return ok ? EXIT_SUCCESS : EXIT_FAILURE; #else - return test_memory_map_noexception() ? EXIT_SUCCESS : EXIT_FAILURE; + std::cout << "padded_memory_map is disabled in this configuration; " + "set SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=ON in CMake to " + "enable it on Windows. Test skipped." << std::endl; + return EXIT_SUCCESS; #endif } -#endif \ No newline at end of file