From 019bf1dfe03145bb5aa7421cbdf8c4fbc91f9afd Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Thu, 12 Mar 2026 14:36:01 -0400 Subject: [PATCH] adding memory-file mapping (#2625) * adding memory-file mapping * tuning * init _capacity * adding missing header * update doc --- doc/basics.md | 15 +++++++ doc/dom.md | 16 +++++++ doc/iterate_many.md | 14 +++++++ doc/parse_many.md | 14 +++++++ include/simdjson/padded_string-inl.h | 60 +++++++++++++++++++++++++++ include/simdjson/padded_string.h | 56 +++++++++++++++++++++++++ include/simdjson/padded_string_view.h | 2 +- tests/CMakeLists.txt | 1 + tests/memory_map_tests.cpp | 47 +++++++++++++++++++++ 9 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 tests/memory_map_tests.cpp diff --git a/doc/basics.md b/doc/basics.md index e884ee27c..bb1ab53c9 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -271,6 +271,21 @@ Further, they may use the AreFileApisANSI function to determine whether the filename is interpreted using the ANSI or the system default OEM codepage, and they may call SetFileApisToOEM accordingly. + +**Advanced feature:** +On non-Windows systems, you can use memory-file mapping to create a `simdjson::padded_string_view` +from a file on disk. + +```cpp +// If the macro _WIN32 is defined, this will not work since we do not support memory-file mapping +// under Windows at this time. +simdjson::padded_memory_map map(myfilename); +if (!map.is_valid()) { /* handle error */ } +simdjson::padded_string_view view = map.view(); // view is usable while padded_memory_map is in scope +ondemand::document doc = parser.iterate(view); // parse the JSON +``` + + Documents are iterators ----------------------- diff --git a/doc/dom.md b/doc/dom.md index bbd8c0748..a19856190 100644 --- a/doc/dom.md +++ b/doc/dom.md @@ -125,6 +125,22 @@ Further, they may use the AreFileApisANSI function to determine whether the filename is interpreted using the ANSI or the system default OEM codepage, and they may call SetFileApisToOEM accordingly. + +**Advanced feature:** +On non-Windows systems, you can use memory-file mapping to create a `simdjson::padded_string_view` +from a file on disk. + +```cpp +// if the macro _WIN32 is defined, this will not work since we do not support Windows +simdjson::padded_memory_map map(TWITTER_JSON); +if (!map.is_valid()) { /* handle error */ } +simdjson::padded_string_view view = map.view(); // view is usable while padded_memory_map is in scope +ondemand::document doc = parser.iterate(view); // parse the JSON +``` + +Using memory-file mapping requires some care. The file should not be modified while you are +accessing it. + Using the Parsed JSON --------------------- diff --git a/doc/iterate_many.md b/doc/iterate_many.md index 77e9553fe..3f04bec65 100644 --- a/doc/iterate_many.md +++ b/doc/iterate_many.md @@ -154,6 +154,20 @@ for (auto doc : docs) { See [basics.md](basics.md#newline-delimited-json-ndjson-and-json-lines) for an overview of the API. + +**Advanced feature:** +On non-Windows systems, you can use memory-file mapping to create a `simdjson::padded_string_view` +from a file on disk. + +```cpp +// If the macro _WIN32 is defined, this will not work since we do not support memory-file mapping +// under Windows at this time. +simdjson::padded_memory_map map(myfilename); +if (!map.is_valid()) { /* handle error */ } +simdjson::padded_string_view view = map.view(); // view is usable while padded_memory_map is in scope +ondemand::document doc = parser.iterate(view); // parse the JSON +``` + ## Use cases From [jsonlines.org](http://jsonlines.org/examples/): diff --git a/doc/parse_many.md b/doc/parse_many.md index a489d5d51..413db12f3 100644 --- a/doc/parse_many.md +++ b/doc/parse_many.md @@ -217,6 +217,20 @@ got full document at 29 ``` + +**Advanced feature:** +On non-Windows systems, you can use memory-file mapping to create a `simdjson::padded_string_view` +from a file on disk. + +```cpp +// If the macro _WIN32 is defined, this will not work since we do not support memory-file mapping +// under Windows at this time. +simdjson::padded_memory_map map(myfilename); +if (!map.is_valid()) { /* handle error */ } +simdjson::padded_string_view view = map.view(); // view is usable while padded_memory_map is in scope +ondemand::document doc = parser.iterate(view); // parse the JSON +``` + Incomplete streams ----------- diff --git a/include/simdjson/padded_string-inl.h b/include/simdjson/padded_string-inl.h index 928814691..dd4371120 100644 --- a/include/simdjson/padded_string-inl.h +++ b/include/simdjson/padded_string-inl.h @@ -10,6 +10,14 @@ #include #include +#ifndef _WIN32 +#include +#include +#include +#include +#include +#endif + namespace simdjson { namespace internal { @@ -366,6 +374,57 @@ inline bool padded_string_builder::reserve(size_t additional) noexcept { return true; } + +#ifndef _WIN32 +simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept { + + int fd = open(filename, O_RDONLY); + if (fd == -1) { + return; // file not found or cannot be opened, data will be nullptr + } + struct stat st; + if (fstat(fd, &st) == -1) { + close(fd); + return; // failed to get file size, data will be nullptr + } + size = (size_t)st.st_size; + size_t total_size = size + simdjson::SIMDJSON_PADDING; + void *anon_map = + mmap(NULL, total_size, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (anon_map == MAP_FAILED) { + close(fd); + return; // failed to create anonymous mapping, data will be nullptr + } + void *file_map = + mmap(anon_map, size, PROT_READ, MAP_SHARED | MAP_FIXED, fd, 0); + if (file_map == MAP_FAILED) { + munmap(anon_map, total_size); + close(fd); + return; // failed to mmap file, data will be nullptr + } + data = (const char *)file_map; + close(fd); // no longer needed after mapping +} + +simdjson_inline padded_memory_map::~padded_memory_map() noexcept { + if (data != nullptr) { + munmap((void *)data, size + simdjson::SIMDJSON_PADDING); + } +} + + +simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noexcept simdjson_lifetime_bound { + if(!is_valid()) { + return simdjson::padded_string_view(); // return an empty view if mapping failed + } + return simdjson::padded_string_view(data, size, size + simdjson::SIMDJSON_PADDING); +} + +simdjson_inline bool padded_memory_map::is_valid() const noexcept { + return data != nullptr; +} +#endif // _WIN32 + } // namespace simdjson inline simdjson::padded_string operator ""_padded(const char *str, size_t len) { @@ -376,4 +435,5 @@ inline simdjson::padded_string operator ""_padded(const char8_t *str, size_t len return simdjson::padded_string(reinterpret_cast(str), len); } #endif + #endif // SIMDJSON_PADDED_STRING_INL_H diff --git a/include/simdjson/padded_string.h b/include/simdjson/padded_string.h index c036c2ead..1e83e5f62 100644 --- a/include/simdjson/padded_string.h +++ b/include/simdjson/padded_string.h @@ -277,6 +277,62 @@ inline std::ostream& operator<<(std::ostream& out, const padded_string& s) { ret inline std::ostream& operator<<(std::ostream& out, simdjson_result &s) noexcept(false) { return out << s.value(); } #endif + +#ifndef _WIN32 +/** + * A class representing a memory-mapped file with padding. + * It is only available on non-Windows platforms, as Windows has different APIs for memory mapping. + */ +class padded_memory_map { +public: + /** + * Create a new padded memory map for the given file. + * After creating the memory map, you can call view() to get a padded_string_view of the file content. + * The memory map will be automatically released when the padded_memory_map instance is destroyed. + * Note that the file content is not copied, so this is efficient for large files. However, + * the file must remain unchanged while the memory map is in use. In case of error (e.g., file not found, + * permission denied, etc.), the memory map will be invalid and view() will return an empty view. + * You can check if the memory map is valid by calling is_valid() before using view(). + * + * @param filename the path to the file to memory-map. + */ + simdjson_inline padded_memory_map(const char *filename) noexcept; + /** + * Destroy the padded memory map and release any resources. + */ + simdjson_inline ~padded_memory_map() noexcept; + + // lifetime of the view is tied to the memory map, so we can return a view + // directly + /** + * Get a view of the memory-mapped file. It always succeeds, but the view may be empty + * if the memory map is invalid (e.g., due to file not found, permission denied, etc.). + * You can check if the memory map is valid by calling is_valid() before using the view. + * + * Lifetime of the view is tied to the memory map, so the view should not be used after the + * padded_memory_map instance is destroyed. + * + * @return a padded_string_view representing the memory-mapped file, or an empty view if the memory map is invalid. + */ + simdjson_inline simdjson::padded_string_view view() const noexcept simdjson_lifetime_bound; + /** + * Check if the memory map is valid. + * + * @return true if the memory map is valid, false otherwise. + */ + simdjson_inline bool is_valid() const noexcept; + +private: + padded_memory_map() = delete; + padded_memory_map(const padded_memory_map &) = delete; + padded_memory_map &operator=(const padded_memory_map &) = delete; + const char *data{nullptr}; + size_t size{0}; +}; +#endif // _WIN32 + + + } // namespace simdjson // This is deliberately outside of simdjson so that people get it without having to use the namespace diff --git a/include/simdjson/padded_string_view.h b/include/simdjson/padded_string_view.h index 4b5fdc983..56c5694c3 100644 --- a/include/simdjson/padded_string_view.h +++ b/include/simdjson/padded_string_view.h @@ -17,7 +17,7 @@ namespace simdjson { */ class padded_string_view : public std::string_view { private: - size_t _capacity; + size_t _capacity{0}; public: /** Create an empty padded_string_view. */ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9cffb049e..855174576 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -11,6 +11,7 @@ link_libraries(simdjson) add_cpp_test(unicode_tests LABELS dom acceptance per_implementation) add_cpp_test(minify_tests LABELS other acceptance per_implementation) add_cpp_test(padded_string_tests LABELS other acceptance ) +add_cpp_test(memory_map_tests LABELS other acceptance ) add_cpp_test(prettify_tests LABELS other acceptance per_implementation) add_cpp_test(fractured_json_tests LABELS other acceptance per_implementation) diff --git a/tests/memory_map_tests.cpp b/tests/memory_map_tests.cpp new file mode 100644 index 000000000..e6b6adb41 --- /dev/null +++ b/tests/memory_map_tests.cpp @@ -0,0 +1,47 @@ +#ifdef _WIN32 +#include +// This test is not supported on Windows because it relies on POSIX APIs like +// mmap. Please run it on a POSIX-compliant system. +int main() { return EXIT_SUCCESS; } +#else + +#include "simdjson.h" +#include "test_macros.h" + +#if SIMDJSON_EXCEPTIONS +bool test_memory_map_exception() { + TEST_START(); + simdjson::padded_memory_map map(TWITTER_JSON); + ASSERT_TRUE(map.is_valid()); + simdjson::padded_string_view view = map.view(); + simdjson::ondemand::document doc = simdjson::ondemand::parser::get_parser().iterate(view); + ASSERT_TRUE(doc["search_metadata"]["count"].get_uint64() == 100); + TEST_SUCCEED(); +} +#endif + +bool test_memory_map_noexception() { + TEST_START(); + simdjson::padded_memory_map map(TWITTER_JSON); + if (!map.is_valid()) { + std::cerr << "Failed to memory-map the file " << TWITTER_JSON << std::endl; + return false; + } + simdjson::padded_string_view view = map.view(); + simdjson::ondemand::parser parser; + simdjson::ondemand::document doc; + ASSERT_SUCCESS( parser.iterate(view).get(doc) ); + uint64_t count; + ASSERT_SUCCESS( doc["search_metadata"]["count"].get(count) ); + ASSERT_EQUAL(count, 100); + TEST_SUCCEED(); +} + +int main() { +#if SIMDJSON_EXCEPTIONS + return (test_memory_map_exception() && test_memory_map_noexception()) ? EXIT_SUCCESS : EXIT_FAILURE; +#else + return test_memory_map_noexception() ? EXIT_SUCCESS : EXIT_FAILURE; +#endif +} +#endif \ No newline at end of file