adding memory-file mapping (#2625)

* adding memory-file mapping

* tuning

* init _capacity

* adding missing header

* update doc

doc/basics.md

@@ -271,6 +271,21 @@ Further, they may use the AreFileApisANSI function to determine whether
 the filename is interpreted using the ANSI or the system default OEM
 codepage, and they may call SetFileApisToOEM accordingly.
 
+
+**Advanced feature:**
+On non-Windows systems, you can use memory-file mapping to create a `simdjson::padded_string_view`
+from a file on disk.
+
+```cpp
+// If the macro _WIN32 is defined, this will not work since we do not support memory-file mapping
+// under Windows at this time.
+simdjson::padded_memory_map map(myfilename);
+if (!map.is_valid()) { /* handle error */ }
+simdjson::padded_string_view view = map.view(); // view is usable while padded_memory_map is in scope
+ondemand::document doc = parser.iterate(view); // parse the JSON
+```
+
+
 Documents are iterators
 -----------------------
 

doc/dom.md

@@ -125,6 +125,22 @@ Further, they may use the AreFileApisANSI function to determine whether
 the filename is interpreted using the ANSI or the system default OEM
 codepage, and they may call SetFileApisToOEM accordingly.
 
+
+**Advanced feature:**
+On non-Windows systems, you can use memory-file mapping to create a `simdjson::padded_string_view`
+from a file on disk.
+
+```cpp
+// if the macro _WIN32 is defined, this will not work since we do not support Windows
+simdjson::padded_memory_map map(TWITTER_JSON);
+if (!map.is_valid()) { /* handle error */ }
+simdjson::padded_string_view view = map.view(); // view is usable while padded_memory_map is in scope
+ondemand::document doc = parser.iterate(view); // parse the JSON
+```
+
+Using memory-file mapping requires some care. The file should not be modified while you are
+accessing it.
+
 Using the Parsed JSON
 ---------------------
 

doc/iterate_many.md

@@ -154,6 +154,20 @@ for (auto doc : docs) {
 
 See [basics.md](basics.md#newline-delimited-json-ndjson-and-json-lines) for an overview of the API.
 
+
+**Advanced feature:**
+On non-Windows systems, you can use memory-file mapping to create a `simdjson::padded_string_view`
+from a file on disk.
+
+```cpp
+// If the macro _WIN32 is defined, this will not work since we do not support memory-file mapping
+// under Windows at this time.
+simdjson::padded_memory_map map(myfilename);
+if (!map.is_valid()) { /* handle error */ }
+simdjson::padded_string_view view = map.view(); // view is usable while padded_memory_map is in scope
+ondemand::document doc = parser.iterate(view); // parse the JSON
+```
+
 ## Use cases
 
 From [jsonlines.org](http://jsonlines.org/examples/):

doc/parse_many.md

@@ -217,6 +217,20 @@ got full document at 29
 ```
 
 
+
+**Advanced feature:**
+On non-Windows systems, you can use memory-file mapping to create a `simdjson::padded_string_view`
+from a file on disk.
+
+```cpp
+// If the macro _WIN32 is defined, this will not work since we do not support memory-file mapping
+// under Windows at this time.
+simdjson::padded_memory_map map(myfilename);
+if (!map.is_valid()) { /* handle error */ }
+simdjson::padded_string_view view = map.view(); // view is usable while padded_memory_map is in scope
+ondemand::document doc = parser.iterate(view); // parse the JSON
+```
+
 Incomplete streams
 -----------
 

include/simdjson/padded_string-inl.h

@@ -10,6 +10,14 @@
 #include <climits>
 #include <cwchar>
 
+#ifndef _WIN32
+#include <fcntl.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#endif
+
 namespace simdjson {
 namespace internal {
 
@@ -366,6 +374,57 @@ inline bool padded_string_builder::reserve(size_t additional) noexcept {
   return true;
 }
 
+
+#ifndef _WIN32
+simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept {
+
+    int fd = open(filename, O_RDONLY);
+    if (fd == -1) {
+      return; // file not found or cannot be opened, data will be nullptr
+    }
+    struct stat st;
+    if (fstat(fd, &st) == -1) {
+      close(fd);
+      return; // failed to get file size, data will be nullptr
+    }
+    size = (size_t)st.st_size;
+    size_t total_size = size + simdjson::SIMDJSON_PADDING;
+    void *anon_map =
+        mmap(NULL, total_size, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (anon_map == MAP_FAILED) {
+      close(fd);
+      return; // failed to create anonymous mapping, data will be nullptr
+    }
+    void *file_map =
+        mmap(anon_map, size, PROT_READ, MAP_SHARED | MAP_FIXED, fd, 0);
+    if (file_map == MAP_FAILED) {
+      munmap(anon_map, total_size);
+      close(fd);
+      return; // failed to mmap file, data will be nullptr
+    }
+    data = (const char *)file_map;
+    close(fd); // no longer needed after mapping
+}
+
+simdjson_inline padded_memory_map::~padded_memory_map() noexcept {
+  if (data != nullptr) {
+    munmap((void *)data, size + simdjson::SIMDJSON_PADDING);
+  }
+}
+
+
+simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noexcept simdjson_lifetime_bound {
+  if(!is_valid()) {
+    return simdjson::padded_string_view(); // return an empty view if mapping failed
+  }
+  return simdjson::padded_string_view(data, size, size + simdjson::SIMDJSON_PADDING);
+}
+
+simdjson_inline bool padded_memory_map::is_valid() const noexcept {
+  return data != nullptr;
+}
+#endif // _WIN32
+
 } // namespace simdjson
 
 inline simdjson::padded_string operator ""_padded(const char *str, size_t len) {
@@ -376,4 +435,5 @@ inline simdjson::padded_string operator ""_padded(const char8_t *str, size_t len
   return simdjson::padded_string(reinterpret_cast<const char *>(str), len);
 }
 #endif
+
 #endif // SIMDJSON_PADDED_STRING_INL_H

include/simdjson/padded_string.h

@@ -277,6 +277,62 @@ inline std::ostream& operator<<(std::ostream& out, const padded_string& s) { ret
 inline std::ostream& operator<<(std::ostream& out, simdjson_result<padded_string> &s) noexcept(false) { return out << s.value(); }
 #endif
 
+
+#ifndef _WIN32
+/**
+ * A class representing a memory-mapped file with padding.
+ * It is only available on non-Windows platforms, as Windows has different APIs for memory mapping.
+ */
+class padded_memory_map {
+public:
+  /**
+   * Create a new padded memory map for the given file.
+   * After creating the memory map, you can call view() to get a padded_string_view of the file content.
+   * The memory map will be automatically released when the padded_memory_map instance is destroyed.
+   * Note that the file content is not copied, so this is efficient for large files. However,
+   * the file must remain unchanged while the memory map is in use. In case of error (e.g., file not found,
+   * permission denied, etc.), the memory map will be invalid and view() will return an empty view.
+   * You can check if the memory map is valid by calling is_valid() before using view().
+   *
+   * @param filename the path to the file to memory-map.
+   */
+  simdjson_inline padded_memory_map(const char *filename) noexcept;
+  /**
+   * Destroy the padded memory map and release any resources.
+   */
+  simdjson_inline ~padded_memory_map() noexcept;
+
+  // lifetime of the view is tied to the memory map, so we can return a view
+  // directly
+  /**
+   * Get a view of the memory-mapped file. It always succeeds, but the view may be empty
+   * if the memory map is invalid (e.g., due to file not found, permission denied, etc.).
+   * You can check if the memory map is valid by calling is_valid() before using the view.
+   *
+   * Lifetime of the view is tied to the memory map, so the view should not be used after the
+   * padded_memory_map instance is destroyed.
+   *
+   * @return a padded_string_view representing the memory-mapped file, or an empty view if the memory map is invalid.
+   */
+  simdjson_inline simdjson::padded_string_view view() const noexcept simdjson_lifetime_bound;
+  /**
+   * Check if the memory map is valid.
+   *
+   * @return true if the memory map is valid, false otherwise.
+   */
+  simdjson_inline bool is_valid() const noexcept;
+
+private:
+  padded_memory_map() = delete;
+  padded_memory_map(const padded_memory_map &) = delete;
+  padded_memory_map &operator=(const padded_memory_map &) = delete;
+  const char *data{nullptr};
+  size_t size{0};
+};
+#endif // _WIN32
+
+
+
 } // namespace simdjson
 
 // This is deliberately outside of simdjson so that people get it without having to use the namespace

include/simdjson/padded_string_view.h

@@ -17,7 +17,7 @@ namespace simdjson {
  */
 class padded_string_view : public std::string_view {
 private:
-  size_t _capacity;
+  size_t _capacity{0};
 
 public:
   /** Create an empty padded_string_view. */

tests/CMakeLists.txt

@@ -11,6 +11,7 @@ link_libraries(simdjson)
 add_cpp_test(unicode_tests       LABELS dom acceptance per_implementation)
 add_cpp_test(minify_tests        LABELS other acceptance per_implementation)
 add_cpp_test(padded_string_tests LABELS other acceptance                   )
+add_cpp_test(memory_map_tests    LABELS other acceptance                   )
 add_cpp_test(prettify_tests      LABELS other acceptance per_implementation)
 add_cpp_test(fractured_json_tests LABELS other acceptance per_implementation)
 

tests/memory_map_tests.cpp

@@ -0,0 +1,47 @@
+#ifdef _WIN32
+#include <cstdlib>
+// This test is not supported on Windows because it relies on POSIX APIs like
+// mmap. Please run it on a POSIX-compliant system.
+int main() { return EXIT_SUCCESS; }
+#else
+
+#include "simdjson.h"
+#include "test_macros.h"
+
+#if SIMDJSON_EXCEPTIONS
+bool test_memory_map_exception() {
+  TEST_START();
+  simdjson::padded_memory_map map(TWITTER_JSON);
+  ASSERT_TRUE(map.is_valid());
+  simdjson::padded_string_view view = map.view();
+  simdjson::ondemand::document doc = simdjson::ondemand::parser::get_parser().iterate(view);
+  ASSERT_TRUE(doc["search_metadata"]["count"].get_uint64() == 100);
+  TEST_SUCCEED();
+}
+#endif
+
+bool test_memory_map_noexception() {
+    TEST_START();
+    simdjson::padded_memory_map map(TWITTER_JSON);
+    if (!map.is_valid()) {
+        std::cerr << "Failed to memory-map the file " << TWITTER_JSON << std::endl;
+        return false;
+    }
+    simdjson::padded_string_view view = map.view();
+    simdjson::ondemand::parser parser;
+    simdjson::ondemand::document doc;
+    ASSERT_SUCCESS( parser.iterate(view).get(doc) );
+    uint64_t count;
+    ASSERT_SUCCESS( doc["search_metadata"]["count"].get(count) );
+    ASSERT_EQUAL(count, 100);
+    TEST_SUCCEED();
+}
+
+int main() {
+#if SIMDJSON_EXCEPTIONS
+    return (test_memory_map_exception() && test_memory_map_noexception()) ? EXIT_SUCCESS : EXIT_FAILURE;
+#else
+    return test_memory_map_noexception() ? EXIT_SUCCESS : EXIT_FAILURE;
+#endif
+}
+#endif