moving the files back to ondemand for now.

include/simdjson/generic/ondemand/amalgamated.h

@@ -45,4 +45,8 @@
 #include "simdjson/generic/ondemand/token_iterator-inl.h"
 #include "simdjson/generic/ondemand/value_iterator-inl.h"
 
+// JSON builder, ideally they should not be part of the ondemand directory
+// but it is convenient for now to have them here.
+#include "simdjson/generic/ondemand/json_string_builder.h"
+#include "simdjson/generic/ondemand/json_string_builder-inl.h"
 

include/simdjson/generic/ondemand/json_string_builder-inl.h

@@ -0,0 +1,54 @@
+/**
+ * This file is part of the builder API. It is temporarily in the ondemand directory
+ * but we will move it to a builder directory later.
+ */
+#ifndef SIMDJSON_GENERIC_BUILDER_INL_H
+
+#ifndef SIMDJSON_CONDITIONAL_INCLUDE
+#define SIMDJSON_GENERIC_BUILDER_INL_H
+#include "simdjson/generic/builder/json_string_builder.h"
+#endif // SIMDJSON_CONDITIONAL_INCLUDE
+
+namespace simdjson {
+namespace SIMDJSON_IMPLEMENTATION {
+namespace builder {
+simdjson_inline string_builder::string_builder(size_t initial_capacity) :
+  buffer(new (std::nothrow) char[initial_capacity]),
+  position(0), capacity(buffer.get() != nullptr ? initial_capacity : 0),
+  is_valid(buffer.get() != nullptr) {}
+    /***
+      char *padded_buffer = new (std::nothrow) char[totalpaddedlength];
+  if (padded_buffer == nullptr) {
+    return nullptr;
+  }
+     */
+simdjson_inline bool string_builder::capacity_check(size_t upcoming_bytes) {
+  if (upcoming_bytes <= capacity - position) { return true; }
+  // check for overflow:
+  if (position + upcoming_bytes < position) { return false; }
+  grow_buffer(std::max(capacity * 2, position + upcoming_bytes));
+  return is_valid;
+}
+
+simdjson_inline void string_builder::grow_buffer(size_t desired_capacity) {
+  if (!is_valid) { return; }
+  std::unique_ptr<char[]> new_buffer(new (std::nothrow) char[desired_capacity]);
+  if (new_buffer.get() == nullptr) {
+    is_valid = false;
+    capacity = 0;
+    return;
+  }
+  memcpy(new_buffer.get(), buffer.get(), position);
+  buffer.swap(new_buffer);
+  capacity = desired_capacity;
+}
+
+simdjson_inline size_t string_builder::size() const {
+  return is_valid ? position : 0;
+}
+
+} // namespace builder
+} // namespace SIMDJSON_IMPLEMENTATION
+} // namespace simdjson
+
+#endif // SIMDJSON_GENERIC_BUILDER_INL_H

include/simdjson/generic/ondemand/json_string_builder.h

@@ -0,0 +1,146 @@
+/**
+ * This file is part of the builder API. It is temporarily in the ondemand directory
+ * but we will move it to a builder directory later.
+ */
+#ifndef SIMDJSON_GENERIC_BUILDER_H
+
+#ifndef SIMDJSON_CONDITIONAL_INCLUDE
+#define SIMDJSON_GENERIC_BUILDER_H
+#include "simdjson/generic/implementation_simdjson_result_base.h"
+#endif // SIMDJSON_CONDITIONAL_INCLUDE
+
+namespace simdjson {
+namespace SIMDJSON_IMPLEMENTATION {
+namespace builder {
+
+//////////////////////////
+// TODO: at the end of the processing, possibly as an optional step, we should
+// validate the UTF-8 of the string.
+/////////////////////////
+/**
+ * A builder for JSON strings representing documents. This is a low-level
+ * builder that is not meant to be used directly by end-users. Though it
+ * supports atomic types (Booleans, strings), it does not support composed
+ * types (arrays and objects).
+ *
+ * Ultimately, this class should support kernel-specific optimizations. E.g.,
+ * it may make use of SIMD instructions to escape strings faster.
+ */
+class string_builder {
+public:
+  simdjson_inline string_builder(size_t initial_capacity);
+
+  /**
+   * Append number (includes Booleans). Booleans are mapped to the strings
+   * false and true. Numbers are converted to strings abiding by the JSON standard.
+   * Floating-point numbers are converted to the shortest string that 'correctly'
+   * represents the number.
+   */
+  template<typename number_type,
+         typename = typename std::enable_if<std::is_arithmetic<number_type>::value>::type>
+  simdjson_inline void append(number_type v) noexcept;
+
+  /**
+   * Append character c.
+   */
+  simdjson_inline void append(char c)  noexcept;
+
+  /**
+   * Append the string 'null'.
+   */
+  simdjson_inline void append_null()  noexcept;
+
+  /**
+   * Clear the content.
+   */
+  simdjson_inline void clear()  noexcept;
+
+  /**
+   * Append the std::string_view, after escaping it.
+   * There is no UTF-8 validation.
+   */
+  simdjson_inline void escape_and_append(std::string_view input)  noexcept;
+
+  /**
+   * Append the std::string_view surrounded by double quotes, after escaping it.
+   * There is no UTF-8 validation.
+   */
+  simdjson_inline void escape_and_append_with_quotes(std::string_view input)  noexcept;
+
+  /**
+   * Append the C string directly, without escaping.
+   * There is no UTF-8 validation.
+   */
+  simdjson_inline void append_raw(const char *c)  noexcept;
+
+  /**
+   * Append the std::string_view directly, without escaping.
+   * There is no UTF-8 validation.
+   */
+  simdjson_inline void append_raw(std::string_view str)  noexcept;
+
+  /**
+   * Append len characters from str.
+   * There is no UTF-8 validation.
+   */
+  simdjson_inline void append_raw(const char *str, size_t len) noexcept;
+#if SIMDJSON_EXCEPTIONS
+  /**
+   * Creates an std::string from the written JSON buffer.
+   * Throws if memory allocation failed
+   */
+  simdjson_inline operator std::string() const noexcept(false);
+
+  /**
+   * Creates an std::string_view from the written JSON buffer.
+   * Throws if memory allocation failed
+   */
+  simdjson_inline operator std::string_view() const noexcept(false);
+#endif
+
+  /**
+   * Returns a view on the written JSON buffer. Returns an error
+   * if memory allocation failed.
+   */
+  simdjson_inline simdjson_result<std::string_view> view() const noexcept;
+
+  /**
+   * Appends the null character to the buffer and returns
+   * a pointer to the beginning of the written JSON buffer.
+   * Returns an error if memory allocation failed.
+   */
+  simdjson_inline simdjson_result<const char *> c_str();
+
+  /**
+   * Returns the current size of the written JSON buffer.
+   * If an error occurred, returns 0.
+   */
+  simdjson_inline size_t size() const;
+
+private:
+  /**
+   * Returns true if we can write at least upcoming_bytes bytes.
+   * The underlying buffer is reallocated if needed. It is designed
+   * to be called before writing to the buffer. It should be fast.
+   */
+  simdjson_inline bool capacity_check(size_t upcoming_bytes);
+
+  /**
+   * Grow the buffer to at least desired_capacity bytes.
+   * If the allocation fails, is_valid is set to false. We expect
+   * that this function would not be repeatedly called.
+   */
+  simdjson_inline void grow_buffer(size_t desired_capacity);
+  std::unique_ptr<char[]> buffer;
+  size_t position;
+  size_t capacity;
+  bool is_valid{true};
+};
+
+
+
+}
+}
+} // namespace simdjson
+
+#endif // SIMDJSON_GENERIC_BUILDER_H

singleheader/amalgamate.py

@@ -10,12 +10,41 @@ import os
 import re
 import shutil
 import datetime
+import json
 from typing import Dict, List, Optional, Set, TextIO, Union, cast
 
+# Check for Python 3, this does not actually work.
 if sys.version_info < (3, 0):
     sys.stdout.write("Sorry, requires Python 3.x or better\n")
     sys.exit(1)
 
+rules = """
+
+We refer your to the HACKING.md file for more information on how the project is organized.
+
+To help understand the error, here are the rules for including files in simdjson:
+
+All implementation-specific files, including arm64.h, arm64/implementation.h and
+arm64/ondemand.h, must be within SIMDJSON_CONDITIONAL_INCLUDE blocks.
+
+Top-level headers must not be included in any SIMDJSON_CONDITIONAL_INCLUDE block.
+
+Generic files must be included only in amalgamator files (arm64.h,
+arm64/implementation.h, arm64/ondemand.h, generic/amalgamated.h).
+
+We fail if an implementation-specific file is included more than once in the same block.
+We fail if a generic file is included more than once per implementation in the same block.
+
+
+Tip: generally, "file" will search the including file's source directory first, then
+the search paths while <file> does it the other way around.
+We prefer to use <> in simdjson headers to avoid accidentally including a file from the
+wrong directory.
+
+The amalgamate.py script checks that all files are included.
+
+"""
+
 SCRIPTPATH = os.path.dirname(os.path.abspath(sys.argv[0]))
 PROJECTPATH = os.path.dirname(SCRIPTPATH)
 print(f"SCRIPTPATH={SCRIPTPATH} PROJECTPATH={PROJECTPATH}")
@@ -62,6 +91,22 @@ class SimdjsonFile:
     def __str__(self):
         return self.include_path
 
+    def dump(self):
+        return {
+            'root': self.root,
+            'include_path': self.include_path,
+            'includes': [include.include_path for include in self.includes],
+            'included_from': [included_from.include_path for included_from in self.included_from],
+            'editor_only_includes': [editor_only_include.include_path for editor_only_include in self.editor_only_includes],
+            'editor_only_included_from': [editor_only_included_from.include_path for editor_only_included_from in self.editor_only_included_from],
+            'processed': self.processed,
+            'dependency_file': self.dependency_file.include_path if self.dependency_file else None,
+            'is_amalgamator': self.is_amalgamator,
+            'implementation': self.implementation,
+        }
+    def json(self):
+        return json.dumps(self.dump(), indent=4, sort_keys=True, ensure_ascii=False)
+
     def __repr__(self):
         return self.include_path
 
@@ -162,20 +207,21 @@ class SimdjsonFile:
 
     def add_include(self, include: 'SimdjsonFile'):
         if self.is_conditional_include:
-            assert include.is_conditional_include, f"{self} cannot include {include} without #ifndef SIMDJSON_CONDITIONAL_INCLUDE."
+            # If I have a dependency file, I can only include something that has a dependency file.
+            assert include.is_conditional_include, f"{self} cannot include {include} without #ifndef SIMDJSON_CONDITIONAL_INCLUDE. {rules}"
             # TODO make sure we only include amalgamated files that are guaranteed to be included with us (or before us)
             # if include.amalgamator_file:
             #     assert include.amalgamator_file == self, f"{self} cannot include {include}: it should be included from {include.amalgamator_file} instead."
         else:
-            assert include.is_amalgamator or not include.is_conditional_include, f"{self} cannot include {include} because it is an amalgamated file."
+            assert include.is_amalgamator or not include.is_conditional_include, f"{self} cannot include {include} because it is an amalgamated file. {rules}"
 
         self.includes.append(include)
         include.included_from.add(self)
 
     def add_editor_only_include(self, include: 'SimdjsonFile'):
-        assert self.is_conditional_include, f"Cannot use #ifndef SIMDJSON_CONDITIONAL_INCLUDE in {self} because it is not an amalgamated file."
+        assert self.is_conditional_include, f"Cannot use #ifndef SIMDJSON_CONDITIONAL_INCLUDE in {self} because it is not an amalgamated file. {rules}"
         if not include.is_conditional_include:
-            assert self.dependency_file, f"{self} cannot include {include} without #ifndef SIMDJSON_CONDITIONAL_INCLUDE."
+            assert self.dependency_file, f"{self} cannot include {include} without #ifndef SIMDJSON_CONDITIONAL_INCLUDE. {rules}"
         # TODO make sure we only include amalgamated files that are guaranteed to be included with us (or before us)
         # elif include.amalgamator_file:
         #     assert self.is_amalgamated_before(self.amalgamator_file), f"{self} cannot include {include}: it should be included from {include.amalgamator_file} instead."
@@ -190,11 +236,11 @@ class SimdjsonFile:
                 if file.dependency_file == self:
                     for editor_only_include in file.editor_only_includes:
                         if not editor_only_include.is_conditional_include:
-                            assert editor_only_include in self.includes, f"{file} includes {editor_only_include}, but it is not included from {self}. It must be added to {self}."
+                            assert editor_only_include in self.includes, f"{file} includes {editor_only_include}, but it is not included from {self}. It must be added to {self}. {rules}"
                             if editor_only_include in extra_include_set:
                                 extra_include_set.remove(editor_only_include)
 
-            assert len(extra_include_set) == 0, f"{self} unnecessarily includes {extra_include_set}. They are not included in the corresponding amalgamated files."
+            assert len(extra_include_set) == 0, f"{self} unnecessarily includes {extra_include_set}. They are not included in the corresponding amalgamated files. {rules}"
 
 class SimdjsonRepository:
     def __init__(self, project_path: str, relative_roots: List[RelativeRoot]):
@@ -320,6 +366,7 @@ class Amalgamator:
 
         assert not self.editor_only_region
         with open(file.absolute_path, 'r') as fid2:
+            print(f"including: {file}")
             for line in fid2:
                 line = line.rstrip('\n')
 
@@ -329,9 +376,9 @@ class Amalgamator:
 
                 # Ignore lines inside #ifndef SIMDJSON_CONDITIONAL_INCLUDE
                 if re.search(r'^#ifndef\s+SIMDJSON_CONDITIONAL_INCLUDE\s*$', line):
-                    assert file.is_conditional_include, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE but is not an amalgamated file!"
+                    assert file.is_conditional_include, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE but is not an amalgamated file! {rules}"
-                    assert self.in_conditional_include_block, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE without a prior #define SIMDJSON_CONDITIONAL_INCLUDE: {self.include_stack}"
+                    assert self.in_conditional_include_block, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE without a prior #define SIMDJSON_CONDITIONAL_INCLUDE: {self.include_stack} {rules}"
-                    assert not self.editor_only_region, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE twice in a row"
+                    assert not self.editor_only_region, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE twice in a row {rules}"
                     self.editor_only_region = True
 
                 # Handle ignored lines (and ending ignore blocks)
@@ -349,7 +396,7 @@ class Amalgamator:
                         self.editor_only_region = False
                     continue
 
-                assert not end_ignore, f"{file} has #endif // SIMDJSON_CONDITIONAL_INCLUDE without #ifndef SIMDJSON_CONDITIONAL_INCLUDE"
+                assert not end_ignore, f"{file} has #endif // SIMDJSON_CONDITIONAL_INCLUDE without #ifndef SIMDJSON_CONDITIONAL_INCLUDE {rules}"
 
                 # Handle #include lines
                 included = re.search(r'^#include\s+["<]([^">]*)[">]', line)
@@ -376,26 +423,26 @@ class Amalgamator:
                     self.implementation = None
                 elif re.search(r'\bSIMDJSON_IMPLEMENTATION\b', line) and file.include_path != IMPLEMENTATION_DETECTION_H:
                     # copy the line, with SIMDJSON_IMPLEMENTATION replace to what it is currently defined to
-                    assert self.implementation, f"Use of SIMDJSON_IMPLEMENTATION while not defined in {file}: {line}"
+                    assert self.implementation, f"Use of SIMDJSON_IMPLEMENTATION while not defined in {file}: {line}\n{rules}"
                     line = re.sub(r'\bSIMDJSON_IMPLEMENTATION\b',self.implementation,line)
 
                 # Handle defining and undefining SIMDJSON_CONDITIONAL_INCLUDE
                 defined = re.search(r'^#define\s+SIMDJSON_CONDITIONAL_INCLUDE\s*$', line)
                 if defined:
-                    assert not file.is_conditional_include, "SIMDJSON_CONDITIONAL_INCLUDE defined in amalgamated file {file}! Not allowed."
+                    assert not file.is_conditional_include, "SIMDJSON_CONDITIONAL_INCLUDE defined in amalgamated file {file}! Not allowed. {rules}"
-                    assert not self.in_conditional_include_block, f"{file} redefines SIMDJSON_CONDITIONAL_INCLUDE"
+                    assert not self.in_conditional_include_block, f"{file} redefines SIMDJSON_CONDITIONAL_INCLUDE {rules}"
                     self.in_conditional_include_block = True
                     self.found_includes_per_conditional_block.clear()
                     self.write(f'/* defining SIMDJSON_CONDITIONAL_INCLUDE */')
                 elif re.search(r'^#undef\s+SIMDJSON_CONDITIONAL_INCLUDE\s*$', line):
-                    assert not file.is_conditional_include, "SIMDJSON_CONDITIONAL_INCLUDE undefined in amalgamated file {file}! Not allowed."
+                    assert not file.is_conditional_include, "SIMDJSON_CONDITIONAL_INCLUDE undefined in amalgamated file {file}! Not allowed. {rules}"
-                    assert self.in_conditional_include_block, f"{file} undefines SIMDJSON_CONDITIONAL_INCLUDE without defining it"
+                    assert self.in_conditional_include_block, f"{file} undefines SIMDJSON_CONDITIONAL_INCLUDE without defining it {rules}"
                     self.write(f'/* undefining SIMDJSON_CONDITIONAL_INCLUDE */')
                     self.in_conditional_include_block = False
 
                 self.write(line)
 
-            assert not self.editor_only_region, f"{file} ended without #endif // SIMDJSON_CONDITIONAL_INCLUDE"
+            assert not self.editor_only_region, f"{file} ended without #endif // SIMDJSON_CONDITIONAL_INCLUDE {rules}"
 
         self.write(f"/* end file {self.file_to_str(file)} */")
 

singleheader/simdjson.cpp

@@ -1,4 +1,4 @@
-/* auto-generated on 2025-06-24 15:13:44 -0400. Do not edit! */
+/* auto-generated on 2025-07-14 11:45:25 -0400. Do not edit! */
 /* including simdjson.cpp:  */
 /* begin file simdjson.cpp */
 #define SIMDJSON_SRC_SIMDJSON_CPP
@@ -298,6 +298,7 @@ using std::size_t;
 #if defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG))
 // If NDEBUG is set, or __OPTIMIZE__ is set, or we are under MSVC in release mode,
 // then do away with asserts and use __assume.
+// We still recommend that our users set NDEBUG in release mode.
 #if SIMDJSON_VISUAL_STUDIO
 #define SIMDJSON_UNREACHABLE() __assume(0)
 #define SIMDJSON_ASSUME(COND) __assume(COND)
@@ -2342,16 +2343,25 @@ namespace std {
 // It could also wrongly set SIMDJSON_DEVELOPMENT_CHECKS (e.g., if the programmer
 // sets _DEBUG in a release build under Visual Studio, or if some compiler fails to
 // set the __OPTIMIZE__ macro).
+// We make it so that if NDEBUG is defined, then SIMDJSON_DEVELOPMENT_CHECKS
+// is not defined, irrespective of the compiler.
+// We recommend that users set NDEBUG in release builds, so that
+// SIMDJSON_DEVELOPMENT_CHECKS is not defined in release builds by default,
+// irrespective of the compiler.
 #ifndef SIMDJSON_DEVELOPMENT_CHECKS
 #ifdef _MSC_VER
 // Visual Studio seems to set _DEBUG for debug builds.
-#ifdef _DEBUG
+// We set SIMDJSON_DEVELOPMENT_CHECKS to 1 if _DEBUG is defined
+// and NDEBUG is not defined.
+#if defined(_DEBUG) && !defined(NDEBUG)
 #define SIMDJSON_DEVELOPMENT_CHECKS 1
 #endif // _DEBUG
 #else // _MSC_VER
 // All other compilers appear to set __OPTIMIZE__ to a positive integer
 // when the compiler is optimizing.
-#ifndef __OPTIMIZE__
+// We only set SIMDJSON_DEVELOPMENT_CHECKS if both __OPTIMIZE__
+// and NDEBUG are not defined.
+#if !defined(__OPTIMIZE__) && !defined(NDEBUG)
 #define SIMDJSON_DEVELOPMENT_CHECKS 1
 #endif // __OPTIMIZE__
 #endif // _MSC_VER