moving the files back to ondemand for now.

This commit is contained in:
Daniel Lemire
2025-07-14 11:46:06 -04:00
parent d987f1871c
commit 8ad452459c
16 changed files with 1822 additions and 62 deletions
-1
View File
@@ -3,7 +3,6 @@
#include "simdjson/arm64/begin.h"
#include "simdjson/generic/ondemand/amalgamated.h"
#include "simdjson/generic/builder/amalgamated.h"
#include "simdjson/arm64/end.h"
#endif // SIMDJSON_ARM64_ONDEMAND_H
-1
View File
@@ -3,7 +3,6 @@
#include "simdjson/fallback/begin.h"
#include "simdjson/generic/ondemand/amalgamated.h"
#include "simdjson/generic/builder/amalgamated.h"
#include "simdjson/fallback/end.h"
#endif // SIMDJSON_FALLBACK_ONDEMAND_H
@@ -1,10 +0,0 @@
// Stuff other things depend on
//...
// All other declarations
#include "simdjson/generic/builder/json_string_builder.h"
// Inline definitions
#include "simdjson/generic/builder/json_string_builder-inl.h"
@@ -1,17 +0,0 @@
#ifndef SIMDJSON_GENERIC_BUILDER_INL_H
#define SIMDJSON_GENERIC_BUILDER_INL_H
#ifndef SIMDJSON_CONDITIONAL_INCLUDE
#define SIMDJSON_GENERIC_ONDEMAND_ARRAY_H
#include "simdjson/generic/implementation_simdjson_result_base.h"
#include "simdjson/generic/builder/json_string_builder.h"
#endif // SIMDJSON_CONDITIONAL_INCLUDE
namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace builder {
}
}
} // namespace simdjson
#endif // SIMDJSON_GENERIC_BUILDER_INL_H
@@ -45,4 +45,8 @@
#include "simdjson/generic/ondemand/token_iterator-inl.h"
#include "simdjson/generic/ondemand/value_iterator-inl.h"
// JSON builder, ideally they should not be part of the ondemand directory
// but it is convenient for now to have them here.
#include "simdjson/generic/ondemand/json_string_builder.h"
#include "simdjson/generic/ondemand/json_string_builder-inl.h"
@@ -0,0 +1,54 @@
/**
* This file is part of the builder API. It is temporarily in the ondemand directory
* but we will move it to a builder directory later.
*/
#ifndef SIMDJSON_GENERIC_BUILDER_INL_H
#ifndef SIMDJSON_CONDITIONAL_INCLUDE
#define SIMDJSON_GENERIC_BUILDER_INL_H
#include "simdjson/generic/builder/json_string_builder.h"
#endif // SIMDJSON_CONDITIONAL_INCLUDE
namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace builder {
simdjson_inline string_builder::string_builder(size_t initial_capacity) :
buffer(new (std::nothrow) char[initial_capacity]),
position(0), capacity(buffer.get() != nullptr ? initial_capacity : 0),
is_valid(buffer.get() != nullptr) {}
/***
char *padded_buffer = new (std::nothrow) char[totalpaddedlength];
if (padded_buffer == nullptr) {
return nullptr;
}
*/
simdjson_inline bool string_builder::capacity_check(size_t upcoming_bytes) {
if (upcoming_bytes <= capacity - position) { return true; }
// check for overflow:
if (position + upcoming_bytes < position) { return false; }
grow_buffer(std::max(capacity * 2, position + upcoming_bytes));
return is_valid;
}
simdjson_inline void string_builder::grow_buffer(size_t desired_capacity) {
if (!is_valid) { return; }
std::unique_ptr<char[]> new_buffer(new (std::nothrow) char[desired_capacity]);
if (new_buffer.get() == nullptr) {
is_valid = false;
capacity = 0;
return;
}
memcpy(new_buffer.get(), buffer.get(), position);
buffer.swap(new_buffer);
capacity = desired_capacity;
}
simdjson_inline size_t string_builder::size() const {
return is_valid ? position : 0;
}
} // namespace builder
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson
#endif // SIMDJSON_GENERIC_BUILDER_INL_H
@@ -1,8 +1,11 @@
/**
* This file is part of the builder API. It is temporarily in the ondemand directory
* but we will move it to a builder directory later.
*/
#ifndef SIMDJSON_GENERIC_BUILDER_H
#define SIMDJSON_GENERIC_BUILDER_H
#ifndef SIMDJSON_CONDITIONAL_INCLUDE
#define SIMDJSON_GENERIC_ONDEMAND_ARRAY_H
#define SIMDJSON_GENERIC_BUILDER_H
#include "simdjson/generic/implementation_simdjson_result_base.h"
#endif // SIMDJSON_CONDITIONAL_INCLUDE
@@ -10,7 +13,10 @@ namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace builder {
//////////////////////////
// TODO: at the end of the processing, possibly as an optional step, we should
// validate the UTF-8 of the string.
/////////////////////////
/**
* A builder for JSON strings representing documents. This is a low-level
* builder that is not meant to be used directly by end-users. Though it
@@ -22,7 +28,7 @@ namespace builder {
*/
class string_builder {
public:
string_builder(size_t initial_capacity);
simdjson_inline string_builder(size_t initial_capacity);
/**
* Append number (includes Booleans). Booleans are mapped to the strings
@@ -124,7 +130,7 @@ private:
* If the allocation fails, is_valid is set to false. We expect
* that this function would not be repeatedly called.
*/
void grow_buffer(size_t desired_capacity);
simdjson_inline void grow_buffer(size_t desired_capacity);
std::unique_ptr<char[]> buffer;
size_t position;
size_t capacity;
-1
View File
@@ -3,7 +3,6 @@
#include "simdjson/haswell/begin.h"
#include "simdjson/generic/ondemand/amalgamated.h"
#include "simdjson/generic/builder/amalgamated.h"
#include "simdjson/haswell/end.h"
#endif // SIMDJSON_HASWELL_ONDEMAND_H
-1
View File
@@ -3,7 +3,6 @@
#include "simdjson/icelake/begin.h"
#include "simdjson/generic/ondemand/amalgamated.h"
#include "simdjson/generic/builder/amalgamated.h"
#include "simdjson/icelake/end.h"
#endif // SIMDJSON_ICELAKE_ONDEMAND_H
-1
View File
@@ -3,7 +3,6 @@
#include "simdjson/lasx/begin.h"
#include "simdjson/generic/ondemand/amalgamated.h"
#include "simdjson/generic/builder/amalgamated.h"
#include "simdjson/lasx/end.h"
#endif // SIMDJSON_LASX_ONDEMAND_H
-1
View File
@@ -3,7 +3,6 @@
#include "simdjson/lsx/begin.h"
#include "simdjson/generic/ondemand/amalgamated.h"
#include "simdjson/generic/builder/amalgamated.h"
#include "simdjson/lsx/end.h"
#endif // SIMDJSON_LSX_ONDEMAND_H
-1
View File
@@ -3,7 +3,6 @@
#include "simdjson/ppc64/begin.h"
#include "simdjson/generic/ondemand/amalgamated.h"
#include "simdjson/generic/builder/amalgamated.h"
#include "simdjson/ppc64/end.h"
#endif // SIMDJSON_PPC64_ONDEMAND_H
-1
View File
@@ -3,7 +3,6 @@
#include "simdjson/westmere/begin.h"
#include "simdjson/generic/ondemand/amalgamated.h"
#include "simdjson/generic/builder/amalgamated.h"
#include "simdjson/westmere/end.h"
#endif // SIMDJSON_WESTMERE_IMPLEMENTATION_H
+63 -16
View File
@@ -10,12 +10,41 @@ import os
import re
import shutil
import datetime
import json
from typing import Dict, List, Optional, Set, TextIO, Union, cast
# Check for Python 3, this does not actually work.
if sys.version_info < (3, 0):
sys.stdout.write("Sorry, requires Python 3.x or better\n")
sys.exit(1)
rules = """
We refer your to the HACKING.md file for more information on how the project is organized.
To help understand the error, here are the rules for including files in simdjson:
All implementation-specific files, including arm64.h, arm64/implementation.h and
arm64/ondemand.h, must be within SIMDJSON_CONDITIONAL_INCLUDE blocks.
Top-level headers must not be included in any SIMDJSON_CONDITIONAL_INCLUDE block.
Generic files must be included only in amalgamator files (arm64.h,
arm64/implementation.h, arm64/ondemand.h, generic/amalgamated.h).
We fail if an implementation-specific file is included more than once in the same block.
We fail if a generic file is included more than once per implementation in the same block.
Tip: generally, "file" will search the including file's source directory first, then
the search paths while <file> does it the other way around.
We prefer to use <> in simdjson headers to avoid accidentally including a file from the
wrong directory.
The amalgamate.py script checks that all files are included.
"""
SCRIPTPATH = os.path.dirname(os.path.abspath(sys.argv[0]))
PROJECTPATH = os.path.dirname(SCRIPTPATH)
print(f"SCRIPTPATH={SCRIPTPATH} PROJECTPATH={PROJECTPATH}")
@@ -62,6 +91,22 @@ class SimdjsonFile:
def __str__(self):
return self.include_path
def dump(self):
return {
'root': self.root,
'include_path': self.include_path,
'includes': [include.include_path for include in self.includes],
'included_from': [included_from.include_path for included_from in self.included_from],
'editor_only_includes': [editor_only_include.include_path for editor_only_include in self.editor_only_includes],
'editor_only_included_from': [editor_only_included_from.include_path for editor_only_included_from in self.editor_only_included_from],
'processed': self.processed,
'dependency_file': self.dependency_file.include_path if self.dependency_file else None,
'is_amalgamator': self.is_amalgamator,
'implementation': self.implementation,
}
def json(self):
return json.dumps(self.dump(), indent=4, sort_keys=True, ensure_ascii=False)
def __repr__(self):
return self.include_path
@@ -162,20 +207,21 @@ class SimdjsonFile:
def add_include(self, include: 'SimdjsonFile'):
if self.is_conditional_include:
assert include.is_conditional_include, f"{self} cannot include {include} without #ifndef SIMDJSON_CONDITIONAL_INCLUDE."
# If I have a dependency file, I can only include something that has a dependency file.
assert include.is_conditional_include, f"{self} cannot include {include} without #ifndef SIMDJSON_CONDITIONAL_INCLUDE. {rules}"
# TODO make sure we only include amalgamated files that are guaranteed to be included with us (or before us)
# if include.amalgamator_file:
# assert include.amalgamator_file == self, f"{self} cannot include {include}: it should be included from {include.amalgamator_file} instead."
else:
assert include.is_amalgamator or not include.is_conditional_include, f"{self} cannot include {include} because it is an amalgamated file."
assert include.is_amalgamator or not include.is_conditional_include, f"{self} cannot include {include} because it is an amalgamated file. {rules}"
self.includes.append(include)
include.included_from.add(self)
def add_editor_only_include(self, include: 'SimdjsonFile'):
assert self.is_conditional_include, f"Cannot use #ifndef SIMDJSON_CONDITIONAL_INCLUDE in {self} because it is not an amalgamated file."
assert self.is_conditional_include, f"Cannot use #ifndef SIMDJSON_CONDITIONAL_INCLUDE in {self} because it is not an amalgamated file. {rules}"
if not include.is_conditional_include:
assert self.dependency_file, f"{self} cannot include {include} without #ifndef SIMDJSON_CONDITIONAL_INCLUDE."
assert self.dependency_file, f"{self} cannot include {include} without #ifndef SIMDJSON_CONDITIONAL_INCLUDE. {rules}"
# TODO make sure we only include amalgamated files that are guaranteed to be included with us (or before us)
# elif include.amalgamator_file:
# assert self.is_amalgamated_before(self.amalgamator_file), f"{self} cannot include {include}: it should be included from {include.amalgamator_file} instead."
@@ -190,11 +236,11 @@ class SimdjsonFile:
if file.dependency_file == self:
for editor_only_include in file.editor_only_includes:
if not editor_only_include.is_conditional_include:
assert editor_only_include in self.includes, f"{file} includes {editor_only_include}, but it is not included from {self}. It must be added to {self}."
assert editor_only_include in self.includes, f"{file} includes {editor_only_include}, but it is not included from {self}. It must be added to {self}. {rules}"
if editor_only_include in extra_include_set:
extra_include_set.remove(editor_only_include)
assert len(extra_include_set) == 0, f"{self} unnecessarily includes {extra_include_set}. They are not included in the corresponding amalgamated files."
assert len(extra_include_set) == 0, f"{self} unnecessarily includes {extra_include_set}. They are not included in the corresponding amalgamated files. {rules}"
class SimdjsonRepository:
def __init__(self, project_path: str, relative_roots: List[RelativeRoot]):
@@ -320,6 +366,7 @@ class Amalgamator:
assert not self.editor_only_region
with open(file.absolute_path, 'r') as fid2:
print(f"including: {file}")
for line in fid2:
line = line.rstrip('\n')
@@ -329,9 +376,9 @@ class Amalgamator:
# Ignore lines inside #ifndef SIMDJSON_CONDITIONAL_INCLUDE
if re.search(r'^#ifndef\s+SIMDJSON_CONDITIONAL_INCLUDE\s*$', line):
assert file.is_conditional_include, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE but is not an amalgamated file!"
assert self.in_conditional_include_block, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE without a prior #define SIMDJSON_CONDITIONAL_INCLUDE: {self.include_stack}"
assert not self.editor_only_region, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE twice in a row"
assert file.is_conditional_include, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE but is not an amalgamated file! {rules}"
assert self.in_conditional_include_block, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE without a prior #define SIMDJSON_CONDITIONAL_INCLUDE: {self.include_stack} {rules}"
assert not self.editor_only_region, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE twice in a row {rules}"
self.editor_only_region = True
# Handle ignored lines (and ending ignore blocks)
@@ -349,7 +396,7 @@ class Amalgamator:
self.editor_only_region = False
continue
assert not end_ignore, f"{file} has #endif // SIMDJSON_CONDITIONAL_INCLUDE without #ifndef SIMDJSON_CONDITIONAL_INCLUDE"
assert not end_ignore, f"{file} has #endif // SIMDJSON_CONDITIONAL_INCLUDE without #ifndef SIMDJSON_CONDITIONAL_INCLUDE {rules}"
# Handle #include lines
included = re.search(r'^#include\s+["<]([^">]*)[">]', line)
@@ -376,26 +423,26 @@ class Amalgamator:
self.implementation = None
elif re.search(r'\bSIMDJSON_IMPLEMENTATION\b', line) and file.include_path != IMPLEMENTATION_DETECTION_H:
# copy the line, with SIMDJSON_IMPLEMENTATION replace to what it is currently defined to
assert self.implementation, f"Use of SIMDJSON_IMPLEMENTATION while not defined in {file}: {line}"
assert self.implementation, f"Use of SIMDJSON_IMPLEMENTATION while not defined in {file}: {line}\n{rules}"
line = re.sub(r'\bSIMDJSON_IMPLEMENTATION\b',self.implementation,line)
# Handle defining and undefining SIMDJSON_CONDITIONAL_INCLUDE
defined = re.search(r'^#define\s+SIMDJSON_CONDITIONAL_INCLUDE\s*$', line)
if defined:
assert not file.is_conditional_include, "SIMDJSON_CONDITIONAL_INCLUDE defined in amalgamated file {file}! Not allowed."
assert not self.in_conditional_include_block, f"{file} redefines SIMDJSON_CONDITIONAL_INCLUDE"
assert not file.is_conditional_include, "SIMDJSON_CONDITIONAL_INCLUDE defined in amalgamated file {file}! Not allowed. {rules}"
assert not self.in_conditional_include_block, f"{file} redefines SIMDJSON_CONDITIONAL_INCLUDE {rules}"
self.in_conditional_include_block = True
self.found_includes_per_conditional_block.clear()
self.write(f'/* defining SIMDJSON_CONDITIONAL_INCLUDE */')
elif re.search(r'^#undef\s+SIMDJSON_CONDITIONAL_INCLUDE\s*$', line):
assert not file.is_conditional_include, "SIMDJSON_CONDITIONAL_INCLUDE undefined in amalgamated file {file}! Not allowed."
assert self.in_conditional_include_block, f"{file} undefines SIMDJSON_CONDITIONAL_INCLUDE without defining it"
assert not file.is_conditional_include, "SIMDJSON_CONDITIONAL_INCLUDE undefined in amalgamated file {file}! Not allowed. {rules}"
assert self.in_conditional_include_block, f"{file} undefines SIMDJSON_CONDITIONAL_INCLUDE without defining it {rules}"
self.write(f'/* undefining SIMDJSON_CONDITIONAL_INCLUDE */')
self.in_conditional_include_block = False
self.write(line)
assert not self.editor_only_region, f"{file} ended without #endif // SIMDJSON_CONDITIONAL_INCLUDE"
assert not self.editor_only_region, f"{file} ended without #endif // SIMDJSON_CONDITIONAL_INCLUDE {rules}"
self.write(f"/* end file {self.file_to_str(file)} */")
+13 -3
View File
@@ -1,4 +1,4 @@
/* auto-generated on 2025-06-24 15:13:44 -0400. Do not edit! */
/* auto-generated on 2025-07-14 11:45:25 -0400. Do not edit! */
/* including simdjson.cpp: */
/* begin file simdjson.cpp */
#define SIMDJSON_SRC_SIMDJSON_CPP
@@ -298,6 +298,7 @@ using std::size_t;
#if defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG))
// If NDEBUG is set, or __OPTIMIZE__ is set, or we are under MSVC in release mode,
// then do away with asserts and use __assume.
// We still recommend that our users set NDEBUG in release mode.
#if SIMDJSON_VISUAL_STUDIO
#define SIMDJSON_UNREACHABLE() __assume(0)
#define SIMDJSON_ASSUME(COND) __assume(COND)
@@ -2342,16 +2343,25 @@ namespace std {
// It could also wrongly set SIMDJSON_DEVELOPMENT_CHECKS (e.g., if the programmer
// sets _DEBUG in a release build under Visual Studio, or if some compiler fails to
// set the __OPTIMIZE__ macro).
// We make it so that if NDEBUG is defined, then SIMDJSON_DEVELOPMENT_CHECKS
// is not defined, irrespective of the compiler.
// We recommend that users set NDEBUG in release builds, so that
// SIMDJSON_DEVELOPMENT_CHECKS is not defined in release builds by default,
// irrespective of the compiler.
#ifndef SIMDJSON_DEVELOPMENT_CHECKS
#ifdef _MSC_VER
// Visual Studio seems to set _DEBUG for debug builds.
#ifdef _DEBUG
// We set SIMDJSON_DEVELOPMENT_CHECKS to 1 if _DEBUG is defined
// and NDEBUG is not defined.
#if defined(_DEBUG) && !defined(NDEBUG)
#define SIMDJSON_DEVELOPMENT_CHECKS 1
#endif // _DEBUG
#else // _MSC_VER
// All other compilers appear to set __OPTIMIZE__ to a positive integer
// when the compiler is optimizing.
#ifndef __OPTIMIZE__
// We only set SIMDJSON_DEVELOPMENT_CHECKS if both __OPTIMIZE__
// and NDEBUG are not defined.
#if !defined(__OPTIMIZE__) && !defined(NDEBUG)
#define SIMDJSON_DEVELOPMENT_CHECKS 1
#endif // __OPTIMIZE__
#endif // _MSC_VER
+1677 -3
View File
File diff suppressed because it is too large Load Diff