diff --git a/CMakeLists.txt b/CMakeLists.txt index 217477b93..af9448de6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,11 +54,17 @@ endif() option(SIMDJSON_FEATURE_ONDEMAND_API "Enable ondemand API" ON) option(SIMDJSON_FEATURE_DOM_API "Enable DOM API" ON) +option(SIMDJSON_FEATURE_BUILDER_API "Enable builder API" ON) -add_compile_definitions( - SIMDJSON_FEATURE_ONDEMAND_API=$ - SIMDJSON_FEATURE_DOM_API=$ -) +if(NOT SIMDJSON_FEATURE_ONDEMAND_API) + add_compile_definitions(SIMDJSON_FEATURE_ONDEMAND_API=0) +endif() +if(NOT SIMDJSON_FEATURE_DOM_API) + add_compile_definitions(SIMDJSON_FEATURE_DOM_API=0) +endif() +if(NOT SIMDJSON_FEATURE_BUILDER_API) + add_compile_definitions(SIMDJSON_FEATURE_BUILDER_API=0) +endif() if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.25.0") option(SIMDJSON_STATIC_REFLECTION "Enables static reflection (experimental), requires C++26" OFF) diff --git a/HACKING.md b/HACKING.md index 026cccf2f..ea56e8610 100644 --- a/HACKING.md +++ b/HACKING.md @@ -34,11 +34,13 @@ By default the library is built in Release mode. ## Optional APIs (Feature Flags) -simdjson provides two APIs: +simdjson provides three APIs: - DOM (`simdjson::dom`) - OnDemand (`simdjson::ondemand`) +- Builder (`simdjson::builder`) — requires DOM You can enable or disable these APIs at compile time to reduce binary size and compile time. +At least one of DOM or OnDemand must be enabled. ### Configuration @@ -46,7 +48,8 @@ Using CMake: ```bash cmake -DSIMDJSON_FEATURE_DOM_API=OFF \ - -DSIMDJSON_FEATURE_ONDEMAND_API=ON .. + -DSIMDJSON_FEATURE_ONDEMAND_API=ON \ + -DSIMDJSON_FEATURE_BUILDER_API=OFF .. Assertions and development checks ------------------------------ diff --git a/include/simdjson.h b/include/simdjson.h index dc81ac2be..df59f6e02 100644 --- a/include/simdjson.h +++ b/include/simdjson.h @@ -46,6 +46,7 @@ #include "simdjson/error.h" #include "simdjson/error-inl.h" #include "simdjson/implementation.h" +#include "simdjson/builtin.h" #include "simdjson/minify.h" #include "simdjson/padded_string.h" #include "simdjson/padded_string-inl.h" @@ -54,7 +55,9 @@ #if SIMDJSON_FEATURE_DOM_API #include "simdjson/dom.h" +#if SIMDJSON_FEATURE_BUILDER_API #include "simdjson/builder.h" +#endif // SIMDJSON_FEATURE_BUILDER_API #endif // SIMDJSON_FEATURE_DOM_API #if SIMDJSON_FEATURE_ONDEMAND_API diff --git a/include/simdjson/feature_macros.h b/include/simdjson/feature_macros.h index a3d8b7d77..e027d7be5 100644 --- a/include/simdjson/feature_macros.h +++ b/include/simdjson/feature_macros.h @@ -8,6 +8,10 @@ #define SIMDJSON_FEATURE_ONDEMAND_API 1 #endif +#if !defined(SIMDJSON_FEATURE_BUILDER_API) +#define SIMDJSON_FEATURE_BUILDER_API 1 +#endif + #if !SIMDJSON_FEATURE_DOM_API && !SIMDJSON_FEATURE_ONDEMAND_API #error "simdjson: at least one of SIMDJSON_FEATURE_DOM_API or SIMDJSON_FEATURE_ONDEMAND_API must be enabled." #endif diff --git a/include/simdjson/generic/ondemand/amalgamated.h b/include/simdjson/generic/ondemand/amalgamated.h index 837b8028f..044925c2c 100644 --- a/include/simdjson/generic/ondemand/amalgamated.h +++ b/include/simdjson/generic/ondemand/amalgamated.h @@ -1,8 +1,10 @@ +#ifndef SIMDJSON_CONDITIONAL_INCLUDE #include "simdjson/feature_macros.h" +#endif // SIMDJSON_CONDITIONAL_INCLUDE #if SIMDJSON_FEATURE_ONDEMAND_API -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_BUILDER_DEPENDENCIES_H) +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_ONDEMAND_DEPENDENCIES_H) #error simdjson/generic/ondemand/dependencies.h must be included before simdjson/generic/ondemand/amalgamated.h! #endif diff --git a/include/simdjson/generic/ondemand/dependencies.h b/include/simdjson/generic/ondemand/dependencies.h index 3c4fdc3a6..3c2b7ed32 100644 --- a/include/simdjson/generic/ondemand/dependencies.h +++ b/include/simdjson/generic/ondemand/dependencies.h @@ -8,6 +8,7 @@ // Internal headers needed for ondemand generics. // All includes not under simdjson/generic/ondemand must be here! // Otherwise, amalgamation will fail. +#include "simdjson/feature_macros.h" #include "simdjson/dom/base.h" // for MINIMAL_DOCUMENT_CAPACITY #include "simdjson/implementation.h" #include "simdjson/padded_string.h" diff --git a/include/simdjson/ondemand.h b/include/simdjson/ondemand.h index 6394fa3fe..d25a10563 100644 --- a/include/simdjson/ondemand.h +++ b/include/simdjson/ondemand.h @@ -8,12 +8,14 @@ namespace simdjson { * @copydoc simdjson::builtin::ondemand */ namespace ondemand = builtin::ondemand; +#if SIMDJSON_FEATURE_BUILDER_API /** * @copydoc simdjson::builtin::builder */ namespace builder = builtin::builder; +#endif // SIMDJSON_FEATURE_BUILDER_API -#if SIMDJSON_STATIC_REFLECTION +#if SIMDJSON_STATIC_REFLECTION && SIMDJSON_FEATURE_BUILDER_API /** * Create a JSON string from any user-defined type using static reflection. * Only available when SIMDJSON_STATIC_REFLECTION is enabled. diff --git a/singleheader/all_features_test.py b/singleheader/all_features_test.py new file mode 100755 index 000000000..38d614686 --- /dev/null +++ b/singleheader/all_features_test.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +""" +Amalgamates simdjson into temporary files, then builds and runs 7 feature-flag +test programs. Reports compile time and binary size for each configuration. +""" + +import os +import sys +import subprocess +import tempfile +import time +import shutil + +SCRIPTPATH = os.path.dirname(os.path.abspath(__file__)) +PROJECTPATH = os.path.dirname(SCRIPTPATH) +TESTSRC = os.path.join(PROJECTPATH, "tests", "feature_flag_tests") + +# ── feature-flag configurations ────────────────────────────────────────────── +CONFIGS = [ + { + "name": "dom_only", + "flags": {"SIMDJSON_FEATURE_DOM_API": 1, + "SIMDJSON_FEATURE_ONDEMAND_API": 0, + "SIMDJSON_FEATURE_BUILDER_API": 0}, + }, + { + "name": "ondemand_only", + "flags": {"SIMDJSON_FEATURE_DOM_API": 1, + "SIMDJSON_FEATURE_ONDEMAND_API": 1, + "SIMDJSON_FEATURE_BUILDER_API": 0}, + }, + { + "name": "dom_and_ondemand", + "flags": {"SIMDJSON_FEATURE_DOM_API": 1, + "SIMDJSON_FEATURE_ONDEMAND_API": 1, + "SIMDJSON_FEATURE_BUILDER_API": 0}, + }, + { + "name": "builder_and_dom", + "flags": {"SIMDJSON_FEATURE_DOM_API": 1, + "SIMDJSON_FEATURE_ONDEMAND_API": 0, + "SIMDJSON_FEATURE_BUILDER_API": 1}, + }, + { + "name": "builder_and_ondemand", + "flags": {"SIMDJSON_FEATURE_DOM_API": 1, + "SIMDJSON_FEATURE_ONDEMAND_API": 1, + "SIMDJSON_FEATURE_BUILDER_API": 1}, + }, + { + "name": "builder_dom_and_ondemand", + "flags": {"SIMDJSON_FEATURE_DOM_API": 1, + "SIMDJSON_FEATURE_ONDEMAND_API": 1, + "SIMDJSON_FEATURE_BUILDER_API": 1}, + }, + { + "name": "builder_only", + "flags": {"SIMDJSON_FEATURE_DOM_API": 1, + "SIMDJSON_FEATURE_ONDEMAND_API": 0, + "SIMDJSON_FEATURE_BUILDER_API": 1}, + }, +] + +CXX = os.environ.get("CXX", "c++") +CXXSTD = os.environ.get("CXXSTD", "-std=c++17") +OPT = os.environ.get("OPT", "-O2") + + +def amalgamate(output_dir): + """Run amalgamate.py, writing simdjson.h / simdjson.cpp into *output_dir*.""" + env = os.environ.copy() + env["AMALGAMATE_OUTPUT_PATH"] = output_dir + cmd = [sys.executable, os.path.join(SCRIPTPATH, "amalgamate.py")] + result = subprocess.run(cmd, env=env, capture_output=True, text=True) + if result.returncode != 0: + print("amalgamate.py failed:") + print(result.stdout) + print(result.stderr) + sys.exit(1) + header = os.path.join(output_dir, "simdjson.h") + source = os.path.join(output_dir, "simdjson.cpp") + assert os.path.isfile(header), f"missing {header}" + assert os.path.isfile(source), f"missing {source}" + return header, source + + +def compile_and_run(name, flags, amal_dir, build_dir): + """ + 1. Compile simdjson.cpp -> simdjson.o (timed, size reported) + 2. Compile test .cpp -> test.o (timed, size reported) + 3. Link and run the test program + + Returns (lib_time, lib_size, test_time, test_size) or Nones on failure. + """ + src = os.path.join(TESTSRC, f"{name}.cpp") + simdjson_cpp = os.path.join(amal_dir, "simdjson.cpp") + simdjson_o = os.path.join(build_dir, f"{name}_simdjson.o") + test_o = os.path.join(build_dir, f"{name}.o") + binary = os.path.join(build_dir, name) + + defs = [f"-D{k}={v}" for k, v in flags.items()] + + # ── compile simdjson.o (timed) ──────────────────────────────────────── + cmd_lib = [CXX, CXXSTD, OPT, "-c"] + defs + [ + f"-I{amal_dir}", "-o", simdjson_o, simdjson_cpp, + ] + t0 = time.monotonic() + result = subprocess.run(cmd_lib, capture_output=True, text=True) + lib_time = time.monotonic() - t0 + + if result.returncode != 0: + print(f" COMPILE FAILED (simdjson.o): {name}") + print(f" command: {' '.join(cmd_lib)}") + print(result.stderr) + return None, None, None, None + + lib_size = os.path.getsize(simdjson_o) + + # ── compile test source (timed) ─────────────────────────────────────── + cmd_test = [CXX, CXXSTD, OPT, "-c"] + defs + [ + f"-I{amal_dir}", "-o", test_o, src, + ] + t0 = time.monotonic() + result = subprocess.run(cmd_test, capture_output=True, text=True) + test_time = time.monotonic() - t0 + + if result.returncode != 0: + print(f" COMPILE FAILED (test): {name}") + print(f" command: {' '.join(cmd_test)}") + print(result.stderr) + return None, None, None, None + + test_size = os.path.getsize(test_o) + + # ── link ────────────────────────────────────────────────────────────── + cmd_link = [CXX, "-o", binary, test_o, simdjson_o] + result = subprocess.run(cmd_link, capture_output=True, text=True) + if result.returncode != 0: + print(f" LINK FAILED: {name}") + print(f" command: {' '.join(cmd_link)}") + print(result.stderr) + return None, None, None, None + + # ── run ─────────────────────────────────────────────────────────────── + result = subprocess.run([binary], capture_output=True, text=True) + if result.returncode != 0: + print(f" RUN FAILED: {name}") + print(result.stdout) + print(result.stderr) + return lib_time, lib_size, test_time, test_size + + print(f" {result.stdout.strip()}") + return lib_time, lib_size, test_time, test_size + + +def human_size(n): + """Return e.g. '1.23 MB' or '456 KB'.""" + if n is None: + return "N/A" + if n >= 1_000_000: + return f"{n / 1_000_000:.2f} MB" + if n >= 1_000: + return f"{n / 1_000:.1f} KB" + return f"{n} B" + + +def main(): + tmpdir = tempfile.mkdtemp(prefix="simdjson_features_") + amal_dir = os.path.join(tmpdir, "amalgamated") + build_dir = os.path.join(tmpdir, "build") + os.makedirs(amal_dir, exist_ok=True) + os.makedirs(build_dir, exist_ok=True) + + print(f"Working directory: {tmpdir}") + print(f"Compiler: {CXX} flags: {CXXSTD} {OPT}") + print() + + # ── amalgamate ──────────────────────────────────────────────────────── + print("Amalgamating...") + amalgamate(amal_dir) + print("Amalgamation done.\n") + + # ── build & run each configuration ──────────────────────────────────── + results = [] + for cfg in CONFIGS: + name = cfg["name"] + flags = cfg["flags"] + enabled = [k.replace("SIMDJSON_FEATURE_", "").replace("_API", "") + for k, v in flags.items() if v] + label = " + ".join(enabled) if enabled else "none" + print(f"[{name}] ({label})") + lib_time, lib_size, test_time, test_size = \ + compile_and_run(name, flags, amal_dir, build_dir) + results.append((name, label, lib_time, lib_size, test_time, test_size)) + + print() + + # ── summary table ───────────────────────────────────────────────────── + name_w = max(len(r[0]) for r in results) + label_w = max(len(r[1]) for r in results) + hdr = (f"{'Test':<{name_w}} {'APIs':<{label_w}}" + f" {'lib time':>9} {'simdjson.o':>11}" + f" {'test time':>10} {'test .o':>11}") + print(hdr) + print("-" * len(hdr)) + for name, label, lt, ls, tt, ts in results: + lt_s = f"{lt:.2f} s" if lt is not None else "FAIL" + ls_s = human_size(ls) + tt_s = f"{tt:.2f} s" if tt is not None else "FAIL" + ts_s = human_size(ts) + print(f"{name:<{name_w}} {label:<{label_w}}" + f" {lt_s:>9} {ls_s:>11}" + f" {tt_s:>10} {ts_s:>11}") + + # ── cleanup ─────────────────────────────────────────────────────────── + shutil.rmtree(tmpdir, ignore_errors=True) + + # exit non-zero if anything failed + if any(lt is None for _, _, lt, _, _, _ in results): + print("\nSome tests FAILED.") + sys.exit(1) + print("\nAll tests passed.") + + +if __name__ == "__main__": + main() diff --git a/singleheader/amalgamate.py b/singleheader/amalgamate.py index da748d570..5db1c2ba4 100755 --- a/singleheader/amalgamate.py +++ b/singleheader/amalgamate.py @@ -268,7 +268,21 @@ class SimdjsonFile: print(f" Adding include: {self} includes {include}") if self.is_conditional_include: # If I have a dependency file, I can only include something that has a dependency file. - assert include.is_conditional_include, f"Error: Amalgamated file '{self}' is trying to include '{include}', but '{include}' is not an amalgamated file. Amalgamated files can only include other amalgamated files to maintain conditional inclusion structure. Check the inclusion rules in the script's 'rules' variable. {rules}" + if not include.is_conditional_include: + dep = self.dependency_file + dep_hint = f" and add it to the dependency file '{dep}'" if dep else "" + raise AssertionError( + f"Error: Amalgamated file '{self}' is trying to include '{include}', " + f"but '{include}' is not an amalgamated file.\n\n" + f"FIX: Wrap the #include \"{include}\" in a conditional block:\n\n" + f" #ifndef SIMDJSON_CONDITIONAL_INCLUDE\n" + f" #include \"{include}\"\n" + f" #endif // SIMDJSON_CONDITIONAL_INCLUDE\n\n" + f"This makes the include editor-only (skipped during amalgamation){dep_hint}.\n\n" + f"During amalgamation, '{include}' is already included earlier in the " + f"amalgamated output, so it does not need to be included again.\n\n" + f"{rules}" + ) # TODO make sure we only include amalgamated files that are guaranteed to be included with us (or before us) # if include.amalgamator_file: # assert include.amalgamator_file == self, f"{self} cannot include {include}: it should be included from {include.amalgamator_file} instead." @@ -425,8 +439,10 @@ class Amalgamator: self.implementation = "SIMDJSON_BUILTIN_IMPLEMENTATION" assert not self.editor_only_region, f"Error: Already in an editor-only region when starting to write '{file}'. Ensure proper nesting of conditional blocks." + editor_only_start_line = None + bare_endif_lines = [] with open(file.absolute_path, 'r') as fid2: - for line in fid2: + for line_number, line in enumerate(fid2, 1): line = line.rstrip('\n') # Ignore #pragma once, it causes warnings if it ends up in a .cpp file @@ -439,6 +455,7 @@ class Amalgamator: assert self.in_conditional_include_block, f"Error: File '{file}' uses '#ifndef SIMDJSON_CONDITIONAL_INCLUDE' without a prior '#define SIMDJSON_CONDITIONAL_INCLUDE'. Ensure the define comes first. Stack: {self.include_stack}. {rules}" assert not self.editor_only_region, f"Error: File '{file}' uses '#ifndef SIMDJSON_CONDITIONAL_INCLUDE' twice in a row. Ensure conditional blocks are properly nested and closed. {rules}" self.editor_only_region = True + editor_only_start_line = line_number # Handle ignored lines (and ending ignore blocks) end_ignore = endif_conditional_re.search(line) @@ -453,6 +470,12 @@ class Amalgamator: file.add_editor_only_include(included_file) if end_ignore: self.editor_only_region = False + editor_only_start_line = None + else: + # Track bare #endif lines that might be the intended closer + stripped = line.strip() + if stripped == '#endif' or (stripped.startswith('#endif') and 'SIMDJSON_CONDITIONAL_INCLUDE' not in stripped): + bare_endif_lines.append((line_number, line.strip())) continue assert not end_ignore, f"Error: File '{file}' has '#endif // SIMDJSON_CONDITIONAL_INCLUDE' without a matching '#ifndef'. Ensure proper conditional block structure. {rules}" @@ -501,7 +524,35 @@ class Amalgamator: self.write(line) - assert not self.editor_only_region, f"Error: File '{file}' ended without closing the '#endif // SIMDJSON_CONDITIONAL_INCLUDE'. Ensure all conditional blocks are properly closed. {rules}" + if self.editor_only_region: + msg = ( + f"Error: File '{file}' ended without closing the " + f"'#endif // SIMDJSON_CONDITIONAL_INCLUDE' block " + f"(opened at line {editor_only_start_line}).\n\n" + ) + if bare_endif_lines: + msg += ( + f"HINT: Found #endif line(s) inside the block that are missing " + f"the required comment. The amalgamation script looks for exactly:\n\n" + f" #endif // SIMDJSON_CONDITIONAL_INCLUDE\n\n" + f"but found:\n" + ) + for ln, text in bare_endif_lines: + msg += f" line {ln}: {text}\n" + msg += ( + f"\nFIX: Change the #endif to:\n\n" + f" #endif // SIMDJSON_CONDITIONAL_INCLUDE\n\n" + f"The '// SIMDJSON_CONDITIONAL_INCLUDE' comment is required " + f"for the amalgamation script to recognize it as the closing " + f"of the conditional block.\n" + ) + else: + msg += ( + f"FIX: Add '#endif // SIMDJSON_CONDITIONAL_INCLUDE' to close " + f"the '#ifndef SIMDJSON_CONDITIONAL_INCLUDE' block.\n" + ) + msg += f"\n{rules}" + raise AssertionError(msg) self.write(f"/* end file {self.file_to_str(file)} */") diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index fb4c6adbe..1557c231f 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,8 +1,28 @@ -/* auto-generated on 2026-04-02 19:14:16 -0400. version 4.6.1 Do not edit! */ +/* auto-generated on 2026-04-09 14:54:35 -0400. version 4.6.1 Do not edit! */ /* including simdjson.cpp: */ /* begin file simdjson.cpp */ #define SIMDJSON_SRC_SIMDJSON_CPP +/* including simdjson/feature_macros.h: #include "simdjson/feature_macros.h" */ +/* begin file simdjson/feature_macros.h */ + +#if !defined(SIMDJSON_FEATURE_DOM_API) +#define SIMDJSON_FEATURE_DOM_API 1 +#endif + +#if !defined(SIMDJSON_FEATURE_ONDEMAND_API) +#define SIMDJSON_FEATURE_ONDEMAND_API 1 +#endif + +#if !defined(SIMDJSON_FEATURE_BUILDER_API) +#define SIMDJSON_FEATURE_BUILDER_API 1 +#endif + +#if !SIMDJSON_FEATURE_DOM_API && !SIMDJSON_FEATURE_ONDEMAND_API +#error "simdjson: at least one of SIMDJSON_FEATURE_DOM_API or SIMDJSON_FEATURE_ONDEMAND_API must be enabled." +#endif +/* end file simdjson/feature_macros.h */ + /* including base.h: #include */ /* begin file base.h */ #ifndef SIMDJSON_SRC_BASE_H @@ -7705,10 +7725,13 @@ extern SIMDJSON_DLLIMPORTEXPORT const uint64_t thintable_epi8[256]; #ifndef SIMDJSON_SRC_GENERIC_DEPENDENCIES_H #define SIMDJSON_SRC_GENERIC_DEPENDENCIES_H +/* skipped duplicate #include "simdjson/feature_macros.h" */ /* skipped duplicate #include */ #endif // SIMDJSON_SRC_GENERIC_DEPENDENCIES_H /* end file generic/dependencies.h */ + +#if SIMDJSON_FEATURE_DOM_API /* including generic/stage1/dependencies.h: #include */ /* begin file generic/stage1/dependencies.h */ #ifndef SIMDJSON_SRC_GENERIC_STAGE1_DEPENDENCIES_H @@ -7906,6 +7929,7 @@ enum class tape_type { #endif // SIMDJSON_SRC_GENERIC_STAGE2_DEPENDENCIES_H /* end file generic/stage2/dependencies.h */ +#endif /* including implementation.cpp: #include */ /* begin file implementation.cpp */ @@ -8861,6 +8885,7 @@ const implementation * builtin_implementation() { #define SIMDJSON_SRC_ARM64_CPP /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ /* amalgamation skipped (editor-only): #include */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -14133,6 +14158,260 @@ bool generic_validate_utf8(const char * input, size_t length) { /* end file generic/stage1/amalgamated.h for arm64 */ /* including generic/stage2/amalgamated.h for arm64: #include */ /* begin file generic/stage2/amalgamated.h for arm64 */ +/* including generic/stage2/stringparsing.h for arm64: #include */ +/* begin file generic/stage2/stringparsing.h for arm64 */ +#include +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses +// It is intended to be included multiple times and compiled multiple times + +namespace simdjson { +namespace arm64 { +namespace { +/// @private +namespace stringparsing { + +// begin copypasta +// These chars yield themselves: " \ / +// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab +// u not handled in this table as it's complex +static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +// handle a unicode codepoint +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, + uint8_t **dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + + } + } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + +// handle a unicode codepoint using the wobbly convention +// https://simonsapin.github.io/wtf-8/ +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + +/** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_ptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } +} + +simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } +} + +} // namespace stringparsing + +} // unnamed namespace +} // namespace arm64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H +/* end file generic/stage2/stringparsing.h for arm64 */ + +#if SIMDJSON_FEATURE_DOM_API + // Stuff other things depend on /* including generic/stage2/base.h for arm64: #include */ /* begin file generic/stage2/base.h for arm64 */ @@ -14731,257 +15010,6 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V #endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H /* end file generic/stage2/json_iterator.h for arm64 */ -/* including generic/stage2/stringparsing.h for arm64: #include */ -/* begin file generic/stage2/stringparsing.h for arm64 */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace arm64 { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for arm64 */ /* including generic/stage2/structural_iterator.h for arm64: #include */ /* begin file generic/stage2/structural_iterator.h for arm64 */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H @@ -15365,6 +15393,8 @@ simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { #endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H /* end file generic/stage2/tape_builder.h for arm64 */ + +#endif // SIMDJSON_FEATURE_DOM_API /* end file generic/stage2/amalgamated.h for arm64 */ // @@ -15496,6 +15526,7 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t return arm64::stage1::generic_validate_utf8(buf,len); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } @@ -15503,6 +15534,7 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } +#endif // SIMDJSON_FEATURE_DOM_API SIMDJSON_NO_SANITIZE_MEMORY simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { @@ -15513,11 +15545,13 @@ simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(con return arm64::stringparsing::parse_wobbly_string(src, dst); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } return stage2(_doc); } +#endif // SIMDJSON_FEATURE_DOM_API } // namespace arm64 } // namespace simdjson @@ -15543,6 +15577,7 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t * #define SIMDJSON_SRC_HASWELL_CPP /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ /* amalgamation skipped (editor-only): #include */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -20525,6 +20560,260 @@ bool generic_validate_utf8(const char * input, size_t length) { /* end file generic/stage1/amalgamated.h for haswell */ /* including generic/stage2/amalgamated.h for haswell: #include */ /* begin file generic/stage2/amalgamated.h for haswell */ +/* including generic/stage2/stringparsing.h for haswell: #include */ +/* begin file generic/stage2/stringparsing.h for haswell */ +#include +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses +// It is intended to be included multiple times and compiled multiple times + +namespace simdjson { +namespace haswell { +namespace { +/// @private +namespace stringparsing { + +// begin copypasta +// These chars yield themselves: " \ / +// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab +// u not handled in this table as it's complex +static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +// handle a unicode codepoint +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, + uint8_t **dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + + } + } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + +// handle a unicode codepoint using the wobbly convention +// https://simonsapin.github.io/wtf-8/ +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + +/** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_ptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } +} + +simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } +} + +} // namespace stringparsing + +} // unnamed namespace +} // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H +/* end file generic/stage2/stringparsing.h for haswell */ + +#if SIMDJSON_FEATURE_DOM_API + // Stuff other things depend on /* including generic/stage2/base.h for haswell: #include */ /* begin file generic/stage2/base.h for haswell */ @@ -21123,257 +21412,6 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V #endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H /* end file generic/stage2/json_iterator.h for haswell */ -/* including generic/stage2/stringparsing.h for haswell: #include */ -/* begin file generic/stage2/stringparsing.h for haswell */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace haswell { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for haswell */ /* including generic/stage2/structural_iterator.h for haswell: #include */ /* begin file generic/stage2/structural_iterator.h for haswell */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H @@ -21757,6 +21795,8 @@ simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { #endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H /* end file generic/stage2/tape_builder.h for haswell */ + +#endif // SIMDJSON_FEATURE_DOM_API /* end file generic/stage2/amalgamated.h for haswell */ // @@ -21885,6 +21925,7 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t return haswell::stage1::generic_validate_utf8(buf,len); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } @@ -21892,6 +21933,7 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } +#endif // SIMDJSON_FEATURE_DOM_API SIMDJSON_NO_SANITIZE_MEMORY simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { @@ -21902,11 +21944,13 @@ simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(con return haswell::stringparsing::parse_wobbly_string(src, dst); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } return stage2(_doc); } +#endif // SIMDJSON_FEATURE_DOM_API } // namespace haswell } // namespace simdjson @@ -21935,6 +21979,7 @@ SIMDJSON_UNTARGET_REGION #define SIMDJSON_SRC_ICELAKE_CPP /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ /* amalgamation skipped (editor-only): #include */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -26912,6 +26957,260 @@ bool generic_validate_utf8(const char * input, size_t length) { /* end file generic/stage1/amalgamated.h for icelake */ /* including generic/stage2/amalgamated.h for icelake: #include */ /* begin file generic/stage2/amalgamated.h for icelake */ +/* including generic/stage2/stringparsing.h for icelake: #include */ +/* begin file generic/stage2/stringparsing.h for icelake */ +#include +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses +// It is intended to be included multiple times and compiled multiple times + +namespace simdjson { +namespace icelake { +namespace { +/// @private +namespace stringparsing { + +// begin copypasta +// These chars yield themselves: " \ / +// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab +// u not handled in this table as it's complex +static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +// handle a unicode codepoint +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, + uint8_t **dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + + } + } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + +// handle a unicode codepoint using the wobbly convention +// https://simonsapin.github.io/wtf-8/ +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + +/** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_ptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } +} + +simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } +} + +} // namespace stringparsing + +} // unnamed namespace +} // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H +/* end file generic/stage2/stringparsing.h for icelake */ + +#if SIMDJSON_FEATURE_DOM_API + // Stuff other things depend on /* including generic/stage2/base.h for icelake: #include */ /* begin file generic/stage2/base.h for icelake */ @@ -27510,257 +27809,6 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V #endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H /* end file generic/stage2/json_iterator.h for icelake */ -/* including generic/stage2/stringparsing.h for icelake: #include */ -/* begin file generic/stage2/stringparsing.h for icelake */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace icelake { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for icelake */ /* including generic/stage2/structural_iterator.h for icelake: #include */ /* begin file generic/stage2/structural_iterator.h for icelake */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H @@ -28144,6 +28192,8 @@ simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { #endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H /* end file generic/stage2/tape_builder.h for icelake */ + +#endif // SIMDJSON_FEATURE_DOM_API /* end file generic/stage2/amalgamated.h for icelake */ #undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER @@ -28315,6 +28365,7 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t return icelake::stage1::generic_validate_utf8(buf,len); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } @@ -28322,6 +28373,7 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } +#endif // SIMDJSON_FEATURE_DOM_API SIMDJSON_NO_SANITIZE_MEMORY simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { @@ -28332,11 +28384,13 @@ simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(con return icelake::stringparsing::parse_wobbly_string(src, dst); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } return stage2(_doc); } +#endif // SIMDJSON_FEATURE_DOM_API } // namespace icelake } // namespace simdjson @@ -28365,6 +28419,7 @@ SIMDJSON_UNTARGET_REGION #define SIMDJSON_SRC_PPC64_CPP /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ /* amalgamation skipped (editor-only): #include */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -33570,6 +33625,260 @@ bool generic_validate_utf8(const char * input, size_t length) { /* end file generic/stage1/amalgamated.h for ppc64 */ /* including generic/stage2/amalgamated.h for ppc64: #include */ /* begin file generic/stage2/amalgamated.h for ppc64 */ +/* including generic/stage2/stringparsing.h for ppc64: #include */ +/* begin file generic/stage2/stringparsing.h for ppc64 */ +#include +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses +// It is intended to be included multiple times and compiled multiple times + +namespace simdjson { +namespace ppc64 { +namespace { +/// @private +namespace stringparsing { + +// begin copypasta +// These chars yield themselves: " \ / +// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab +// u not handled in this table as it's complex +static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +// handle a unicode codepoint +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, + uint8_t **dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + + } + } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + +// handle a unicode codepoint using the wobbly convention +// https://simonsapin.github.io/wtf-8/ +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + +/** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_ptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } +} + +simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } +} + +} // namespace stringparsing + +} // unnamed namespace +} // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H +/* end file generic/stage2/stringparsing.h for ppc64 */ + +#if SIMDJSON_FEATURE_DOM_API + // Stuff other things depend on /* including generic/stage2/base.h for ppc64: #include */ /* begin file generic/stage2/base.h for ppc64 */ @@ -34168,257 +34477,6 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V #endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H /* end file generic/stage2/json_iterator.h for ppc64 */ -/* including generic/stage2/stringparsing.h for ppc64: #include */ -/* begin file generic/stage2/stringparsing.h for ppc64 */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace ppc64 { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for ppc64 */ /* including generic/stage2/structural_iterator.h for ppc64: #include */ /* begin file generic/stage2/structural_iterator.h for ppc64 */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H @@ -34802,6 +34860,8 @@ simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { #endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H /* end file generic/stage2/tape_builder.h for ppc64 */ + +#endif // SIMDJSON_FEATURE_DOM_API /* end file generic/stage2/amalgamated.h for ppc64 */ // @@ -34903,6 +34963,7 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t return ppc64::stage1::generic_validate_utf8(buf,len); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } @@ -34910,6 +34971,7 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } +#endif // SIMDJSON_FEATURE_DOM_API SIMDJSON_NO_SANITIZE_MEMORY simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { @@ -34920,11 +34982,13 @@ simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(con return ppc64::stringparsing::parse_wobbly_string(src, dst); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } return stage2(_doc); } +#endif // SIMDJSON_FEATURE_DOM_API } // namespace ppc64 } // namespace simdjson @@ -34950,6 +35014,7 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t * #define SIMDJSON_SRC_WESTMERE_CPP /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ /* amalgamation skipped (editor-only): #include */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -40790,6 +40855,260 @@ bool generic_validate_utf8(const char * input, size_t length) { /* end file generic/stage1/amalgamated.h for westmere */ /* including generic/stage2/amalgamated.h for westmere: #include */ /* begin file generic/stage2/amalgamated.h for westmere */ +/* including generic/stage2/stringparsing.h for westmere: #include */ +/* begin file generic/stage2/stringparsing.h for westmere */ +#include +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses +// It is intended to be included multiple times and compiled multiple times + +namespace simdjson { +namespace westmere { +namespace { +/// @private +namespace stringparsing { + +// begin copypasta +// These chars yield themselves: " \ / +// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab +// u not handled in this table as it's complex +static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +// handle a unicode codepoint +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, + uint8_t **dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + + } + } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + +// handle a unicode codepoint using the wobbly convention +// https://simonsapin.github.io/wtf-8/ +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + +/** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_ptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } +} + +simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } +} + +} // namespace stringparsing + +} // unnamed namespace +} // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H +/* end file generic/stage2/stringparsing.h for westmere */ + +#if SIMDJSON_FEATURE_DOM_API + // Stuff other things depend on /* including generic/stage2/base.h for westmere: #include */ /* begin file generic/stage2/base.h for westmere */ @@ -41388,257 +41707,6 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V #endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H /* end file generic/stage2/json_iterator.h for westmere */ -/* including generic/stage2/stringparsing.h for westmere: #include */ -/* begin file generic/stage2/stringparsing.h for westmere */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace westmere { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for westmere */ /* including generic/stage2/structural_iterator.h for westmere: #include */ /* begin file generic/stage2/structural_iterator.h for westmere */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H @@ -42022,6 +42090,8 @@ simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { #endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H /* end file generic/stage2/tape_builder.h for westmere */ + +#endif // SIMDJSON_FEATURE_DOM_API /* end file generic/stage2/amalgamated.h for westmere */ // @@ -42155,6 +42225,7 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t return westmere::stage1::generic_validate_utf8(buf,len); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } @@ -42162,6 +42233,7 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } +#endif // SIMDJSON_FEATURE_DOM_API SIMDJSON_NO_SANITIZE_MEMORY simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { @@ -42172,11 +42244,13 @@ simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(con return westmere::stringparsing::parse_wobbly_string(src, dst); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } return stage2(_doc); } +#endif // SIMDJSON_FEATURE_DOM_API } // namespace westmere } // namespace simdjson @@ -42204,6 +42278,7 @@ SIMDJSON_UNTARGET_REGION #define SIMDJSON_SRC_LASX_CPP /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ /* amalgamation skipped (editor-only): #include */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -47041,6 +47116,260 @@ bool generic_validate_utf8(const char * input, size_t length) { /* end file generic/stage1/amalgamated.h for lasx */ /* including generic/stage2/amalgamated.h for lasx: #include */ /* begin file generic/stage2/amalgamated.h for lasx */ +/* including generic/stage2/stringparsing.h for lasx: #include */ +/* begin file generic/stage2/stringparsing.h for lasx */ +#include +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses +// It is intended to be included multiple times and compiled multiple times + +namespace simdjson { +namespace lasx { +namespace { +/// @private +namespace stringparsing { + +// begin copypasta +// These chars yield themselves: " \ / +// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab +// u not handled in this table as it's complex +static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +// handle a unicode codepoint +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, + uint8_t **dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + + } + } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + +// handle a unicode codepoint using the wobbly convention +// https://simonsapin.github.io/wtf-8/ +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + +/** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_ptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } +} + +simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } +} + +} // namespace stringparsing + +} // unnamed namespace +} // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H +/* end file generic/stage2/stringparsing.h for lasx */ + +#if SIMDJSON_FEATURE_DOM_API + // Stuff other things depend on /* including generic/stage2/base.h for lasx: #include */ /* begin file generic/stage2/base.h for lasx */ @@ -47639,257 +47968,6 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V #endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H /* end file generic/stage2/json_iterator.h for lasx */ -/* including generic/stage2/stringparsing.h for lasx: #include */ -/* begin file generic/stage2/stringparsing.h for lasx */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace lasx { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for lasx */ /* including generic/stage2/structural_iterator.h for lasx: #include */ /* begin file generic/stage2/structural_iterator.h for lasx */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H @@ -48273,6 +48351,8 @@ simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { #endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H /* end file generic/stage2/tape_builder.h for lasx */ + +#endif // SIMDJSON_FEATURE_DOM_API /* end file generic/stage2/amalgamated.h for lasx */ // @@ -48364,6 +48444,7 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t return lasx::stage1::generic_validate_utf8(buf,len); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } @@ -48371,6 +48452,7 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } +#endif // SIMDJSON_FEATURE_DOM_API SIMDJSON_NO_SANITIZE_MEMORY simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { @@ -48381,11 +48463,13 @@ simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(con return lasx::stringparsing::parse_wobbly_string(src, dst); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } return stage2(_doc); } +#endif // SIMDJSON_FEATURE_DOM_API } // namespace lasx } // namespace simdjson @@ -48418,6 +48502,7 @@ SIMDJSON_UNTARGET_REGION #define SIMDJSON_SRC_LSX_CPP /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ /* amalgamation skipped (editor-only): #include */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -53196,6 +53281,260 @@ bool generic_validate_utf8(const char * input, size_t length) { /* end file generic/stage1/amalgamated.h for lsx */ /* including generic/stage2/amalgamated.h for lsx: #include */ /* begin file generic/stage2/amalgamated.h for lsx */ +/* including generic/stage2/stringparsing.h for lsx: #include */ +/* begin file generic/stage2/stringparsing.h for lsx */ +#include +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses +// It is intended to be included multiple times and compiled multiple times + +namespace simdjson { +namespace lsx { +namespace { +/// @private +namespace stringparsing { + +// begin copypasta +// These chars yield themselves: " \ / +// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab +// u not handled in this table as it's complex +static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +// handle a unicode codepoint +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, + uint8_t **dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + + } + } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + +// handle a unicode codepoint using the wobbly convention +// https://simonsapin.github.io/wtf-8/ +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + +/** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_ptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } +} + +simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } +} + +} // namespace stringparsing + +} // unnamed namespace +} // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H +/* end file generic/stage2/stringparsing.h for lsx */ + +#if SIMDJSON_FEATURE_DOM_API + // Stuff other things depend on /* including generic/stage2/base.h for lsx: #include */ /* begin file generic/stage2/base.h for lsx */ @@ -53794,257 +54133,6 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V #endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H /* end file generic/stage2/json_iterator.h for lsx */ -/* including generic/stage2/stringparsing.h for lsx: #include */ -/* begin file generic/stage2/stringparsing.h for lsx */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace lsx { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for lsx */ /* including generic/stage2/structural_iterator.h for lsx: #include */ /* begin file generic/stage2/structural_iterator.h for lsx */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H @@ -54428,6 +54516,8 @@ simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { #endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H /* end file generic/stage2/tape_builder.h for lsx */ + +#endif // SIMDJSON_FEATURE_DOM_API /* end file generic/stage2/amalgamated.h for lsx */ // @@ -54523,6 +54613,7 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t return lsx::stage1::generic_validate_utf8(buf,len); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } @@ -54530,6 +54621,7 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } +#endif // SIMDJSON_FEATURE_DOM_API SIMDJSON_NO_SANITIZE_MEMORY simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { @@ -54540,11 +54632,13 @@ simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(con return lsx::stringparsing::parse_wobbly_string(src, dst); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } return stage2(_doc); } +#endif // SIMDJSON_FEATURE_DOM_API } // namespace lsx } // namespace simdjson @@ -54570,6 +54664,7 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t * #define SIMDJSON_SRC_RVV_VLS_CPP /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ /* amalgamation skipped (editor-only): #include */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -59770,6 +59865,260 @@ bool generic_validate_utf8(const char * input, size_t length) { /* end file generic/stage1/amalgamated.h for rvv_vls */ /* including generic/stage2/amalgamated.h for rvv_vls: #include */ /* begin file generic/stage2/amalgamated.h for rvv_vls */ +/* including generic/stage2/stringparsing.h for rvv_vls: #include */ +/* begin file generic/stage2/stringparsing.h for rvv_vls */ +#include +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses +// It is intended to be included multiple times and compiled multiple times + +namespace simdjson { +namespace rvv_vls { +namespace { +/// @private +namespace stringparsing { + +// begin copypasta +// These chars yield themselves: " \ / +// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab +// u not handled in this table as it's complex +static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +// handle a unicode codepoint +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, + uint8_t **dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + + } + } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + if(!allow_replacement) { return false; } + code_point = substitution_code_point; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + +// handle a unicode codepoint using the wobbly convention +// https://simonsapin.github.io/wtf-8/ +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + + +/** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_ptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } +} + +simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } +} + +} // namespace stringparsing + +} // unnamed namespace +} // namespace rvv_vls +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H +/* end file generic/stage2/stringparsing.h for rvv_vls */ + +#if SIMDJSON_FEATURE_DOM_API + // Stuff other things depend on /* including generic/stage2/base.h for rvv_vls: #include */ /* begin file generic/stage2/base.h for rvv_vls */ @@ -60368,257 +60717,6 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V #endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H /* end file generic/stage2/json_iterator.h for rvv_vls */ -/* including generic/stage2/stringparsing.h for rvv_vls: #include */ -/* begin file generic/stage2/stringparsing.h for rvv_vls */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace rvv_vls { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for rvv_vls */ /* including generic/stage2/structural_iterator.h for rvv_vls: #include */ /* begin file generic/stage2/structural_iterator.h for rvv_vls */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H @@ -61002,6 +61100,8 @@ simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { #endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H /* end file generic/stage2/tape_builder.h for rvv_vls */ + +#endif // SIMDJSON_FEATURE_DOM_API /* end file generic/stage2/amalgamated.h for rvv_vls */ // @@ -61091,6 +61191,7 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t return rvv_vls::stage1::generic_validate_utf8(buf,len); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } @@ -61098,6 +61199,7 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } +#endif // SIMDJSON_FEATURE_DOM_API SIMDJSON_NO_SANITIZE_MEMORY simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { @@ -61108,11 +61210,13 @@ simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(con return rvv_vls::stringparsing::parse_wobbly_string(src, dst); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } return stage2(_doc); } +#endif // SIMDJSON_FEATURE_DOM_API } // namespace rvv_vls } // namespace simdjson @@ -61137,6 +61241,7 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t * #define SIMDJSON_SRC_FALLBACK_CPP /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ /* amalgamation skipped (editor-only): #include */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -63985,6 +64090,8 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for fallback */ + +#if SIMDJSON_FEATURE_DOM_API /* including generic/stage2/logger.h for fallback: #include */ /* begin file generic/stage2/logger.h for fallback */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H @@ -64870,6 +64977,7 @@ simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { #endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H /* end file generic/stage2/tape_builder.h for fallback */ +#endif // SIMDJSON_FEATURE_DOM_API // // Stage 1 @@ -65291,6 +65399,7 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t namespace simdjson { namespace fallback { +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } @@ -65298,6 +65407,7 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } +#endif // SIMDJSON_FEATURE_DOM_API SIMDJSON_NO_SANITIZE_MEMORY simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { @@ -65308,11 +65418,13 @@ simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(con return fallback::stringparsing::parse_wobbly_string(src, dst); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } return stage2(_doc); } +#endif // SIMDJSON_FEATURE_DOM_API } // namespace fallback } // namespace simdjson @@ -65334,5 +65446,4 @@ simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t * #undef SIMDJSON_CONDITIONAL_INCLUDE SIMDJSON_POP_DISABLE_UNUSED_WARNINGS - /* end file simdjson.cpp */ diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index 7a5d7f17f..ee0a94ae9 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on 2026-04-02 19:14:16 -0400. version 4.6.1 Do not edit! */ +/* auto-generated on 2026-04-09 14:54:35 -0400. version 4.6.1 Do not edit! */ /* including simdjson.h: */ /* begin file simdjson.h */ #ifndef SIMDJSON_H @@ -2527,6 +2527,25 @@ namespace std { #endif #endif // SIMDJSON_COMMON_DEFS_H /* end file simdjson/common_defs.h */ +/* including simdjson/feature_macros.h: #include "simdjson/feature_macros.h" */ +/* begin file simdjson/feature_macros.h */ + +#if !defined(SIMDJSON_FEATURE_DOM_API) +#define SIMDJSON_FEATURE_DOM_API 1 +#endif + +#if !defined(SIMDJSON_FEATURE_ONDEMAND_API) +#define SIMDJSON_FEATURE_ONDEMAND_API 1 +#endif + +#if !defined(SIMDJSON_FEATURE_BUILDER_API) +#define SIMDJSON_FEATURE_BUILDER_API 1 +#endif + +#if !SIMDJSON_FEATURE_DOM_API && !SIMDJSON_FEATURE_ONDEMAND_API +#error "simdjson: at least one of SIMDJSON_FEATURE_DOM_API or SIMDJSON_FEATURE_ONDEMAND_API must be enabled." +#endif +/* end file simdjson/feature_macros.h */ // This provides the public API for simdjson. // DOM and ondemand are amalgamated separately, in simdjson.h @@ -4068,8250 +4087,6 @@ extern SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr& get_ #endif // SIMDJSON_IMPLEMENTATION_H /* end file simdjson/implementation.h */ -/* including simdjson/minify.h: #include "simdjson/minify.h" */ -/* begin file simdjson/minify.h */ -#ifndef SIMDJSON_MINIFY_H -#define SIMDJSON_MINIFY_H - -/* skipped duplicate #include "simdjson/base.h" */ -/* including simdjson/padded_string.h: #include "simdjson/padded_string.h" */ -/* begin file simdjson/padded_string.h */ -#ifndef SIMDJSON_PADDED_STRING_H -#define SIMDJSON_PADDED_STRING_H - -/* skipped duplicate #include "simdjson/base.h" */ -/* skipped duplicate #include "simdjson/error.h" */ - -/* skipped duplicate #include "simdjson/error-inl.h" */ - -#include -#include -#include -#include - -namespace simdjson { - -class padded_string_view; - -/** - * String with extra allocation for ease of use with parser::parse() - * - * This is a move-only class, it cannot be copied. - */ -struct padded_string final { - - /** - * Create a new, empty padded string. - */ - explicit inline padded_string() noexcept; - /** - * Create a new padded string buffer. - * - * @param length the size of the string. - */ - explicit inline padded_string(size_t length) noexcept; - /** - * Create a new padded string by copying the given input. - * - * @param data the buffer to copy - * @param length the number of bytes to copy - */ - explicit inline padded_string(const char *data, size_t length) noexcept; -#ifdef __cpp_char8_t - explicit inline padded_string(const char8_t *data, size_t length) noexcept; -#endif - /** - * Create a new padded string by copying the given input. - * - * @param str_ the string to copy - */ - inline padded_string(const std::string & str_ ) noexcept; - /** - * Create a new padded string by copying the given input. - * - * @param sv_ the string to copy - */ - inline padded_string(std::string_view sv_) noexcept; - /** - * Move one padded string into another. - * - * The original padded string will be reduced to zero capacity. - * - * @param o the string to move. - */ - inline padded_string(padded_string &&o) noexcept; - /** - * Move one padded string into another. - * - * The original padded string will be reduced to zero capacity. - * - * @param o the string to move. - */ - inline padded_string &operator=(padded_string &&o) noexcept; - inline void swap(padded_string &o) noexcept; - ~padded_string() noexcept; - - /** - * The length of the string. - * - * Does not include padding. - */ - size_t size() const noexcept; - - /** - * The length of the string. - * - * Does not include padding. - */ - size_t length() const noexcept; - - /** - * The string data. - **/ - const char *data() const noexcept; - const uint8_t *u8data() const noexcept { return static_cast(static_cast(data_ptr));} - - /** - * The string data. - **/ - char *data() noexcept; - - /** - * Append data to the padded string. Return true on success, false on failure. - * The complexity is O(n) where n is the new size of the string. If you are - * doing multiple appends, consider using padded_string_builder for better performance. - * - * @param data the buffer to append - * @param length the number of bytes to append - */ - inline bool append(const char *data, size_t length) noexcept; - - /** - * Create a std::string_view with the same content. - */ - operator std::string_view() const; - - /** - * Create a padded_string_view with the same content. - */ - operator padded_string_view() const noexcept; - - /** - * Load this padded string from a file. - * - * ## Windows and Unicode - * - * Windows users who need to read files with non-ANSI characters in the - * name should set their code page to UTF-8 (65001) before calling this - * function. This should be the default with Windows 11 and better. - * Further, they may use the AreFileApisANSI function to determine whether - * the filename is interpreted using the ANSI or the system default OEM - * codepage, and they may call SetFileApisToOEM accordingly. - * - * @return IO_ERROR on error. Be mindful that on some 32-bit systems, - * the file size might be limited to 2 GB. - * - * @param path the path to the file. - **/ - inline static simdjson_result load(std::string_view path) noexcept; - - #if defined(_WIN32) && SIMDJSON_CPLUSPLUS17 - /** - * This function accepts a wide string path (UTF-16) and converts it to - * UTF-8 before loading the file. This allows windows users to work - * with unicode file paths without manually converting the paths every time. - * - * @return IO_ERROR on error, including conversion failures. - * - * @param path the path to the file as a wide string. - **/ - inline static simdjson_result load(std::wstring_view path) noexcept; - #endif - -private: - friend class padded_string_builder; - padded_string &operator=(const padded_string &o) = delete; - padded_string(const padded_string &o) = delete; - - size_t viable_size{0}; - char *data_ptr{nullptr}; - -}; // padded_string - -/** - * Builder for constructing padded_string incrementally. - * - * This class allows efficient appending of data and then building a padded_string. - */ -class padded_string_builder { -public: - /** - * Create a new, empty padded string builder. - */ - inline padded_string_builder() noexcept; - - /** - * Create a new padded string builder with initial capacity. - * - * @param capacity the initial capacity of the builder. - */ - inline padded_string_builder(size_t capacity) noexcept; - - /** - * Move constructor. - */ - inline padded_string_builder(padded_string_builder &&o) noexcept; - - /** - * Move assignment. - */ - inline padded_string_builder &operator=(padded_string_builder &&o) noexcept; - - /** - * Copy constructor (deleted). - */ - padded_string_builder(const padded_string_builder &) = delete; - - /** - * Copy assignment (deleted). - */ - padded_string_builder &operator=(const padded_string_builder &) = delete; - - /** - * Destructor. - */ - inline ~padded_string_builder() noexcept; - - /** - * Append data to the builder. - * - * @param newdata the buffer to append - * @param length the number of bytes to append - * @return true if the append succeeded, false if allocation failed - */ - inline bool append(const char *newdata, size_t length) noexcept; - - /** - * Append a string view to the builder. - * - * @param sv the string view to append - * @return true if the append succeeded, false if allocation failed - */ - inline bool append(std::string_view sv) noexcept; - - /** - * Get the current length of the built string. - */ - inline size_t length() const noexcept; - - /** - * Build a padded_string from the current content. The builder's content - * is not modified. If you want to avoid the copy, use convert() instead. - * - * @return a padded_string containing a copy of the built content. - */ - inline padded_string build() const noexcept; - - /** - * Convert the current content into a padded_string. The - * builder's content is emptied, the capacity is lost. - * - * @return a padded_string containing the built content. - */ - inline padded_string convert() noexcept; -private: - size_t size{0}; - size_t capacity{0}; - char *data{nullptr}; - - /** - * Ensure the builder has enough capacity. - * - * @param additional the additional capacity needed. - * @return true if the reservation succeeded, false if allocation failed - */ - inline bool reserve(size_t additional) noexcept; -}; - -/** - * Send padded_string instance to an output stream. - * - * @param out The output stream. - * @param s The padded_string instance. - * @throw if there is an error with the underlying output stream. simdjson itself will not throw. - */ -inline std::ostream& operator<<(std::ostream& out, const padded_string& s) { return out << s.data(); } - -#if SIMDJSON_EXCEPTIONS -/** - * Send padded_string instance to an output stream. - * - * @param out The output stream. - * @param s The padded_string instance. - * @throw simdjson_error if the result being printed has an error. If there is an error with the - * underlying output stream, that error will be propagated (simdjson_error will not be - * thrown). - */ -inline std::ostream& operator<<(std::ostream& out, simdjson_result &s) noexcept(false) { return out << s.value(); } -#endif - - -#ifndef _WIN32 -/** - * A class representing a memory-mapped file with padding. - * It is only available on non-Windows platforms, as Windows has different APIs for memory mapping. - */ -class padded_memory_map { -public: - /** - * Create a new padded memory map for the given file. - * After creating the memory map, you can call view() to get a padded_string_view of the file content. - * The memory map will be automatically released when the padded_memory_map instance is destroyed. - * Note that the file content is not copied, so this is efficient for large files. However, - * the file must remain unchanged while the memory map is in use. In case of error (e.g., file not found, - * permission denied, etc.), the memory map will be invalid and view() will return an empty view. - * You can check if the memory map is valid by calling is_valid() before using view(). - * - * @param filename the path to the file to memory-map. - */ - simdjson_inline padded_memory_map(const char *filename) noexcept; - /** - * Destroy the padded memory map and release any resources. - */ - simdjson_inline ~padded_memory_map() noexcept; - - // lifetime of the view is tied to the memory map, so we can return a view - // directly - /** - * Get a view of the memory-mapped file. It always succeeds, but the view may be empty - * if the memory map is invalid (e.g., due to file not found, permission denied, etc.). - * You can check if the memory map is valid by calling is_valid() before using the view. - * - * Lifetime of the view is tied to the memory map, so the view should not be used after the - * padded_memory_map instance is destroyed. - * - * @return a padded_string_view representing the memory-mapped file, or an empty view if the memory map is invalid. - */ - simdjson_inline simdjson::padded_string_view view() const noexcept simdjson_lifetime_bound; - /** - * Check if the memory map is valid. - * - * @return true if the memory map is valid, false otherwise. - */ - simdjson_inline bool is_valid() const noexcept; - -private: - padded_memory_map() = delete; - padded_memory_map(const padded_memory_map &) = delete; - padded_memory_map &operator=(const padded_memory_map &) = delete; - const char *data{nullptr}; - size_t size{0}; -}; -#endif // _WIN32 - - - -} // namespace simdjson - -// This is deliberately outside of simdjson so that people get it without having to use the namespace -inline simdjson::padded_string operator ""_padded(const char *str, size_t len); -#ifdef __cpp_char8_t -inline simdjson::padded_string operator ""_padded(const char8_t *str, size_t len); -#endif - -namespace simdjson { -namespace internal { - -// The allocate_padded_buffer function is a low-level function to allocate memory -// with padding so we can read past the "length" bytes safely. It is used by -// the padded_string class automatically. It returns nullptr in case -// of error: the caller should check for a null pointer. -// The length parameter is the maximum size in bytes of the string. -// The caller is responsible to free the memory (e.g., delete[] (...)). -inline char *allocate_padded_buffer(size_t length) noexcept; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_PADDED_STRING_H -/* end file simdjson/padded_string.h */ -#include -#include -#include - -namespace simdjson { - -/** - * - * Minify the input string assuming that it represents a JSON string, does not parse or validate. - * This function is much faster than parsing a JSON string and then writing a minified version of it. - * However, it does not validate the input. It will merely return an error in simple cases (e.g., if - * there is a string that was never terminated). - * - * - * @param buf the json document to minify. - * @param len the length of the json document. - * @param dst the buffer to write the minified document to. *MUST* be allocated up to len bytes. - * @param dst_len the number of bytes written. Output only. - * @return the error code, or SUCCESS if there was no error. - */ -simdjson_warn_unused error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept; - -} // namespace simdjson - -#endif // SIMDJSON_MINIFY_H -/* end file simdjson/minify.h */ -/* skipped duplicate #include "simdjson/padded_string.h" */ -/* including simdjson/padded_string-inl.h: #include "simdjson/padded_string-inl.h" */ -/* begin file simdjson/padded_string-inl.h */ -#ifndef SIMDJSON_PADDED_STRING_INL_H -#define SIMDJSON_PADDED_STRING_INL_H - -/* skipped duplicate #include "simdjson/padded_string.h" */ -/* including simdjson/padded_string_view.h: #include "simdjson/padded_string_view.h" */ -/* begin file simdjson/padded_string_view.h */ -#ifndef SIMDJSON_PADDED_STRING_VIEW_H -#define SIMDJSON_PADDED_STRING_VIEW_H - -/* skipped duplicate #include "simdjson/portability.h" */ -/* skipped duplicate #include "simdjson/base.h" // for SIMDJSON_PADDING */ -/* skipped duplicate #include "simdjson/error.h" */ - -#include -#include -#include -#include - -namespace simdjson { - -/** - * User-provided string that promises it has extra padded bytes at the end for use with parser::parse(). - */ -class padded_string_view : public std::string_view { -private: - size_t _capacity{0}; - -public: - /** Create an empty padded_string_view. */ - inline padded_string_view() noexcept = default; - - /** - * Promise the given buffer has at least SIMDJSON_PADDING extra bytes allocated to it. - * - * @param s The string. - * @param len The length of the string (not including padding). - * @param capacity The allocated length of the string, including padding. If the capacity is less - * than the length, the capacity will be set to the length. - */ - explicit inline padded_string_view(const char* s, size_t len, size_t capacity) noexcept; - /** overload explicit inline padded_string_view(const char* s, size_t len) noexcept */ - explicit inline padded_string_view(const uint8_t* s, size_t len, size_t capacity) noexcept; -#ifdef __cpp_char8_t - explicit inline padded_string_view(const char8_t* s, size_t len, size_t capacity) noexcept; -#endif - /** - * Promise the given string has at least SIMDJSON_PADDING extra bytes allocated to it. - * - * The capacity of the string will be used to determine its padding. - * - * @param s The string. - */ - explicit inline padded_string_view(const std::string &s) noexcept; - - /** - * Promise the given string_view has at least SIMDJSON_PADDING extra bytes allocated to it. - * - * @param s The string. - * @param capacity The allocated length of the string, including padding. If the capacity is less - * than the length, the capacity will be set to the length. - */ - explicit inline padded_string_view(std::string_view s, size_t capacity) noexcept; - - /** The number of allocated bytes. */ - inline size_t capacity() const noexcept; - - /** check that the view has sufficient padding */ - inline bool has_sufficient_padding() const noexcept; - - /** - * Remove the UTF-8 Byte Order Mark (BOM) if it exists. - * - * @return whether a BOM was found and removed - */ - inline bool remove_utf8_bom() noexcept; - - /** The amount of padding on the string (capacity() - length()) */ - inline size_t padding() const noexcept; - -}; // padded_string_view - -#if SIMDJSON_EXCEPTIONS -/** - * Send padded_string instance to an output stream. - * - * @param out The output stream. - * @param s The padded_string_view. - * @throw simdjson_error if the result being printed has an error. If there is an error with the - * underlying output stream, that error will be propagated (simdjson_error will not be - * thrown). - */ -inline std::ostream& operator<<(std::ostream& out, simdjson_result &s) noexcept(false); -#endif - -/** - * Create a padded_string_view from a string. The string will be padded with up to SIMDJSON_PADDING - * space characters. The resulting padded_string_view will have a length equal to the original - * string, except maybe for trailing white space characters. - * - * @param s The string. - * @return The padded string. - */ -inline padded_string_view pad(std::string& s) noexcept; - -/** - * Create a padded_string_view from a string. The capacity of the string will be padded with SIMDJSON_PADDING - * characters. The resulting padded_string_view will have a length equal to the original - * string. - * - * @param s The string. - * @return The padded string. - */ -inline padded_string_view pad_with_reserve(std::string& s) noexcept; -} // namespace simdjson - -#endif // SIMDJSON_PADDED_STRING_VIEW_H -/* end file simdjson/padded_string_view.h */ - -/* skipped duplicate #include "simdjson/error-inl.h" */ -/* including simdjson/padded_string_view-inl.h: #include "simdjson/padded_string_view-inl.h" */ -/* begin file simdjson/padded_string_view-inl.h */ -#ifndef SIMDJSON_PADDED_STRING_VIEW_INL_H -#define SIMDJSON_PADDED_STRING_VIEW_INL_H - -/* skipped duplicate #include "simdjson/padded_string_view.h" */ -/* skipped duplicate #include "simdjson/error-inl.h" */ - -#include /* memcmp */ - -namespace simdjson { - -inline padded_string_view::padded_string_view(const char* s, size_t len, size_t capacity) noexcept - : std::string_view(s, len), _capacity(capacity) -{ - if(_capacity < len) { _capacity = len; } -} - -inline padded_string_view::padded_string_view(const uint8_t* s, size_t len, size_t capacity) noexcept - : padded_string_view(reinterpret_cast(s), len, capacity) -{ -} -#ifdef __cpp_char8_t -inline padded_string_view::padded_string_view(const char8_t* s, size_t len, size_t capacity) noexcept - : padded_string_view(reinterpret_cast(s), len, capacity) -{ -} -#endif -inline padded_string_view::padded_string_view(const std::string &s) noexcept - : std::string_view(s), _capacity(s.capacity()) -{ -} - -inline padded_string_view::padded_string_view(std::string_view s, size_t capacity) noexcept - : std::string_view(s), _capacity(capacity) -{ - if(_capacity < s.length()) { _capacity = s.length(); } -} - -inline bool padded_string_view::has_sufficient_padding() const noexcept { - if (padding() >= SIMDJSON_PADDING) { - return true; - } - size_t missing_padding = SIMDJSON_PADDING - padding(); - if(length() < missing_padding) { return false; } - - for (size_t i = length() - missing_padding; i < length(); i++) { - char c = data()[i]; - if (c != ' ' && c != '\t' && c != '\n' && c != '\r') { - return false; - } - } - return true; -} - -inline size_t padded_string_view::capacity() const noexcept { return _capacity; } - -inline size_t padded_string_view::padding() const noexcept { return capacity() - length(); } - -inline bool padded_string_view::remove_utf8_bom() noexcept { - if(length() < 3) { return false; } - if (std::memcmp(data(), "\xEF\xBB\xBF", 3) == 0) { - remove_prefix(3); - _capacity -= 3; - return true; - } - return false; -} - -#if SIMDJSON_EXCEPTIONS -inline std::ostream& operator<<(std::ostream& out, simdjson_result &s) noexcept(false) { return out << s.value(); } -#endif - -inline padded_string_view pad(std::string& s) noexcept { - size_t existing_padding = 0; - for (size_t i = s.size(); i > 0; i--) { - char c = s[i - 1]; - if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { - existing_padding++; - } else { - break; - } - } - size_t needed_padding = 0; - if (existing_padding < SIMDJSON_PADDING) { - needed_padding = SIMDJSON_PADDING - existing_padding; - s.append(needed_padding, ' '); - } - - return padded_string_view(s.data(), s.size() - needed_padding, s.size()); -} - -inline padded_string_view pad_with_reserve(std::string& s) noexcept { - if (s.capacity() - s.size() < SIMDJSON_PADDING) { - s.reserve(s.size() + SIMDJSON_PADDING ); - } - return padded_string_view(s.data(), s.size(), s.capacity()); -} - - - -} // namespace simdjson - - -#endif // SIMDJSON_PADDED_STRING_VIEW_INL_H -/* end file simdjson/padded_string_view-inl.h */ - -#include -#include - -#ifndef _WIN32 -#include -#include -#include -#include -#include -#endif - -namespace simdjson { -namespace internal { - -// The allocate_padded_buffer function is a low-level function to allocate memory -// with padding so we can read past the "length" bytes safely. It is used by -// the padded_string class automatically. It returns nullptr in case -// of error: the caller should check for a null pointer. -// The length parameter is the maximum size in bytes of the string. -// The caller is responsible to free the memory (e.g., delete[] (...)). -inline char *allocate_padded_buffer(size_t length) noexcept { - const size_t totalpaddedlength = length + SIMDJSON_PADDING; - if(totalpaddedlength(1UL<<20)) { - return nullptr; - } -#endif - - char *padded_buffer = new (std::nothrow) char[totalpaddedlength]; - if (padded_buffer == nullptr) { - return nullptr; - } - // We write nulls in the padded region to avoid having uninitialized - // content which may trigger warning for some sanitizers - std::memset(padded_buffer + length, 0, totalpaddedlength - length); - return padded_buffer; -} // allocate_padded_buffer() - -} // namespace internal - - -inline padded_string::padded_string() noexcept = default; -inline padded_string::padded_string(size_t length) noexcept - : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { -} -inline padded_string::padded_string(const char *data, size_t length) noexcept - : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { - if ((data != nullptr) && (data_ptr != nullptr)) { - std::memcpy(data_ptr, data, length); - } - if (data_ptr == nullptr) { - viable_size = 0; - } -} -#ifdef __cpp_char8_t -inline padded_string::padded_string(const char8_t *data, size_t length) noexcept - : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { - if ((data != nullptr) && (data_ptr != nullptr)) { - std::memcpy(data_ptr, reinterpret_cast(data), length); - } - if (data_ptr == nullptr) { - viable_size = 0; - } -} -#endif -// note: do not pass std::string arguments by value -inline padded_string::padded_string(const std::string & str_ ) noexcept - : viable_size(str_.size()), data_ptr(internal::allocate_padded_buffer(str_.size())) { - if (data_ptr == nullptr) { - viable_size = 0; - } else { - std::memcpy(data_ptr, str_.data(), str_.size()); - } -} -// note: do pass std::string_view arguments by value -inline padded_string::padded_string(std::string_view sv_) noexcept - : viable_size(sv_.size()), data_ptr(internal::allocate_padded_buffer(sv_.size())) { - if(simdjson_unlikely(!data_ptr)) { - //allocation failed or zero size - viable_size = 0; - return; - } - if (sv_.size()) { - std::memcpy(data_ptr, sv_.data(), sv_.size()); - } -} -inline padded_string::padded_string(padded_string &&o) noexcept - : viable_size(o.viable_size), data_ptr(o.data_ptr) { - o.data_ptr = nullptr; // we take ownership - o.viable_size = 0; -} - -inline padded_string &padded_string::operator=(padded_string &&o) noexcept { - delete[] data_ptr; - data_ptr = o.data_ptr; - viable_size = o.viable_size; - o.data_ptr = nullptr; // we take ownership - o.viable_size = 0; - return *this; -} - -inline void padded_string::swap(padded_string &o) noexcept { - size_t tmp_viable_size = viable_size; - char *tmp_data_ptr = data_ptr; - viable_size = o.viable_size; - data_ptr = o.data_ptr; - o.data_ptr = tmp_data_ptr; - o.viable_size = tmp_viable_size; -} - -inline padded_string::~padded_string() noexcept { - delete[] data_ptr; -} - -inline size_t padded_string::size() const noexcept { return viable_size; } - -inline size_t padded_string::length() const noexcept { return viable_size; } - -inline const char *padded_string::data() const noexcept { return data_ptr; } - -inline char *padded_string::data() noexcept { return data_ptr; } - -inline bool padded_string::append(const char *data, size_t length) noexcept { - if (length == 0) { - return true; // Nothing to append - } - size_t new_size = viable_size + length; - if (new_size < viable_size) { - // Overflow, cannot append - return false; - } - char *new_data_ptr = internal::allocate_padded_buffer(new_size); - if (new_data_ptr == nullptr) { - // Allocation failed, cannot append - return false; - } - // Copy existing data - if (viable_size > 0) { - std::memcpy(new_data_ptr, data_ptr, viable_size); - } - // Copy new data - std::memcpy(new_data_ptr + viable_size, data, length); - // Update - delete[] data_ptr; - data_ptr = new_data_ptr; - viable_size = new_size; - return true; -} - -inline padded_string::operator std::string_view() const simdjson_lifetime_bound { return std::string_view(data(), length()); } - -inline padded_string::operator padded_string_view() const noexcept simdjson_lifetime_bound { - return padded_string_view(data(), length(), length() + SIMDJSON_PADDING); -} - -inline simdjson_result padded_string::load(std::string_view filename) noexcept { - // std::string_view is not guaranteed to be null-terminated, but std::fopen requires - // a null-terminated C string. Construct a temporary std::string to ensure null-termination. - const std::string null_terminated_filename(filename); - // Open the file - SIMDJSON_PUSH_DISABLE_WARNINGS - SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe - std::FILE *fp = std::fopen(null_terminated_filename.c_str(), "rb"); - SIMDJSON_POP_DISABLE_WARNINGS - - if (fp == nullptr) { - return IO_ERROR; - } - - // Get the file size - int ret; -#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS - ret = _fseeki64(fp, 0, SEEK_END); -#else - ret = std::fseek(fp, 0, SEEK_END); -#endif // _WIN64 - if(ret < 0) { - std::fclose(fp); - return IO_ERROR; - } -#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS - __int64 llen = _ftelli64(fp); - if(llen == -1L) { - std::fclose(fp); - return IO_ERROR; - } -#else - long llen = std::ftell(fp); - if((llen < 0) || (llen == LONG_MAX)) { - std::fclose(fp); - return IO_ERROR; - } -#endif - - // Allocate the padded_string - size_t len = static_cast(llen); - padded_string s(len); - if (s.data() == nullptr) { - std::fclose(fp); - return MEMALLOC; - } - - // Read the padded_string - std::rewind(fp); - size_t bytes_read = std::fread(s.data(), 1, len, fp); - if (std::fclose(fp) != 0 || bytes_read != len) { - return IO_ERROR; - } - - return s; -} - -#if defined(_WIN32) && SIMDJSON_CPLUSPLUS17 -inline simdjson_result padded_string::load(std::wstring_view filename) noexcept { - // std::wstring_view is not guaranteed to be null-terminated, but _wfopen requires - // a null-terminated wide C string. Construct a temporary std::wstring to ensure null-termination. - const std::wstring null_terminated_filename(filename); - // Open the file using the wide characters - SIMDJSON_PUSH_DISABLE_WARNINGS - SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe - std::FILE *fp = _wfopen(null_terminated_filename.c_str(), L"rb"); - SIMDJSON_POP_DISABLE_WARNINGS - - if (fp == nullptr) { - return IO_ERROR; - } - - // Get the file size - int ret; -#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS - ret = _fseeki64(fp, 0, SEEK_END); -#else - ret = std::fseek(fp, 0, SEEK_END); -#endif // _WIN64 - if(ret < 0) { - std::fclose(fp); - return IO_ERROR; - } -#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS - __int64 llen = _ftelli64(fp); - if(llen == -1L) { - std::fclose(fp); - return IO_ERROR; - } -#else - long llen = std::ftell(fp); - if((llen < 0) || (llen == LONG_MAX)) { - std::fclose(fp); - return IO_ERROR; - } -#endif - - // Allocate the padded_string - size_t len = static_cast(llen); - padded_string s(len); - if (s.data() == nullptr) { - std::fclose(fp); - return MEMALLOC; - } - - // Read the padded_string - std::rewind(fp); - size_t bytes_read = std::fread(s.data(), 1, len, fp); - if (std::fclose(fp) != 0 || bytes_read != len) { - return IO_ERROR; - } - - return s; -} -#endif - -// padded_string_builder implementations - -inline padded_string_builder::padded_string_builder() noexcept = default; - -inline padded_string_builder::padded_string_builder(size_t new_capacity) noexcept { - if (new_capacity > 0) { - data = internal::allocate_padded_buffer(new_capacity); - if (data != nullptr) { - this->capacity = new_capacity; - } - } -} - -inline padded_string_builder::padded_string_builder(padded_string_builder &&o) noexcept - : size(o.size), capacity(o.capacity), data(o.data) { - o.size = 0; - o.capacity = 0; - o.data = nullptr; -} - -inline padded_string_builder &padded_string_builder::operator=(padded_string_builder &&o) noexcept { - if (this != &o) { - delete[] data; - size = o.size; - capacity = o.capacity; - data = o.data; - o.size = 0; - o.capacity = 0; - o.data = nullptr; - } - return *this; -} - -inline padded_string_builder::~padded_string_builder() noexcept { - delete[] data; -} - -inline bool padded_string_builder::append(const char *newdata, size_t length) noexcept { - if (length == 0) { - return true; - } - if (!reserve(length)) { - return false; - } - std::memcpy(data + size, newdata, length); - size += length; - return true; -} - -inline bool padded_string_builder::append(std::string_view sv) noexcept { - return append(sv.data(), sv.size()); -} - -inline size_t padded_string_builder::length() const noexcept { - return size; -} - -inline padded_string padded_string_builder::build() const noexcept { - return padded_string(data, size); -} - -inline padded_string padded_string_builder::convert() noexcept { - padded_string result{}; - result.data_ptr = data; - result.viable_size = size; - data = nullptr; - size = 0; - capacity = 0; - return result; -} - -inline bool padded_string_builder::reserve(size_t additional) noexcept { - if (simdjson_unlikely(additional + size < size)) { - return false; // overflow: cannot satisfy request - } - size_t needed = size + additional; - if (needed <= capacity) { - return true; - } - size_t new_capacity = needed; - // We are going to grow the capacity exponentially to avoid - // repeated allocations. - if (new_capacity < 4096) { - new_capacity *= 2; - // overflow guard: ensure new_capacity + new_capacity/2 does not overflow - } else if (new_capacity + new_capacity / 2 > new_capacity) { - new_capacity += new_capacity / 2; // grow by 1.5x - } - char *new_data = internal::allocate_padded_buffer(new_capacity); - if (new_data == nullptr) { - return false; // Allocation failed - } - if (size > 0) { - std::memcpy(new_data, data, size); - } - delete[] data; - data = new_data; - capacity = new_capacity; - return true; -} - - -#ifndef _WIN32 -simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept { - - int fd = open(filename, O_RDONLY); - if (fd == -1) { - return; // file not found or cannot be opened, data will be nullptr - } - struct stat st; - if (fstat(fd, &st) == -1) { - close(fd); - return; // failed to get file size, data will be nullptr - } - size = static_cast(st.st_size); - size_t total_size = size + simdjson::SIMDJSON_PADDING; - void *anon_map = - mmap(NULL, total_size, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (anon_map == MAP_FAILED) { - close(fd); - return; // failed to create anonymous mapping, data will be nullptr - } - void *file_map = - mmap(anon_map, size, PROT_READ, MAP_SHARED | MAP_FIXED, fd, 0); - if (file_map == MAP_FAILED) { - munmap(anon_map, total_size); - close(fd); - return; // failed to mmap file, data will be nullptr - } - data = static_cast(file_map); - close(fd); // no longer needed after mapping -} - -simdjson_inline padded_memory_map::~padded_memory_map() noexcept { - if (data != nullptr) { - munmap(const_cast(data), size + simdjson::SIMDJSON_PADDING); - } -} - - -simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noexcept simdjson_lifetime_bound { - if(!is_valid()) { - return simdjson::padded_string_view(); // return an empty view if mapping failed - } - return simdjson::padded_string_view(data, size, size + simdjson::SIMDJSON_PADDING); -} - -simdjson_inline bool padded_memory_map::is_valid() const noexcept { - return data != nullptr; -} -#endif // _WIN32 - -} // namespace simdjson - -inline simdjson::padded_string operator ""_padded(const char *str, size_t len) { - return simdjson::padded_string(str, len); -} -#ifdef __cpp_char8_t -inline simdjson::padded_string operator ""_padded(const char8_t *str, size_t len) { - return simdjson::padded_string(reinterpret_cast(str), len); -} -#endif - -#endif // SIMDJSON_PADDED_STRING_INL_H -/* end file simdjson/padded_string-inl.h */ -/* skipped duplicate #include "simdjson/padded_string_view.h" */ -/* skipped duplicate #include "simdjson/padded_string_view-inl.h" */ - -/* including simdjson/dom.h: #include "simdjson/dom.h" */ -/* begin file simdjson/dom.h */ -#ifndef SIMDJSON_DOM_H -#define SIMDJSON_DOM_H - -/* including simdjson/dom/base.h: #include "simdjson/dom/base.h" */ -/* begin file simdjson/dom/base.h */ -#ifndef SIMDJSON_DOM_BASE_H -#define SIMDJSON_DOM_BASE_H - -/* skipped duplicate #include "simdjson/base.h" */ - -namespace simdjson { - -/** - * @brief A DOM API on top of the simdjson parser. - */ -namespace dom { - -/** The default batch size for parser.parse_many() and parser.load_many() */ -static constexpr size_t DEFAULT_BATCH_SIZE = 1000000; -/** - * Some adversary might try to set the batch size to 0 or 1, which might cause problems. - * We set a minimum of 32B since anything else is highly likely to be an error. In practice, - * most users will want a much larger batch size. - * - * All non-negative MINIMAL_BATCH_SIZE values should be 'safe' except that, obviously, no JSON - * document can ever span 0 or 1 byte and that very large values would create memory allocation issues. - */ -static constexpr size_t MINIMAL_BATCH_SIZE = 32; - -/** - * It is wasteful to allocate memory for tiny documents (e.g., 4 bytes). - */ -static constexpr size_t MINIMAL_DOCUMENT_CAPACITY = 32; - -class array; -class document; -class document_stream; -class element; -class key_value_pair; -class object; -class parser; - -#ifdef SIMDJSON_THREADS_ENABLED -struct stage1_worker; -#endif // SIMDJSON_THREADS_ENABLED - -} // namespace dom - -namespace internal { - -template -class string_builder; -class tape_ref; - -} // namespace internal - -} // namespace simdjson - -#endif // SIMDJSON_DOM_BASE_H -/* end file simdjson/dom/base.h */ -/* including simdjson/dom/array.h: #include "simdjson/dom/array.h" */ -/* begin file simdjson/dom/array.h */ -#ifndef SIMDJSON_DOM_ARRAY_H -#define SIMDJSON_DOM_ARRAY_H - -#include - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* including simdjson/internal/tape_ref.h: #include "simdjson/internal/tape_ref.h" */ -/* begin file simdjson/internal/tape_ref.h */ -#ifndef SIMDJSON_INTERNAL_TAPE_REF_H -#define SIMDJSON_INTERNAL_TAPE_REF_H - -/* skipped duplicate #include "simdjson/base.h" */ - -namespace simdjson { -namespace dom { -class document; -} // namespace dom - -namespace internal { - -/** - * A reference to an element on the tape. Internal only. - */ -class tape_ref { -public: - simdjson_inline tape_ref() noexcept; - simdjson_inline tape_ref(const dom::document *doc, size_t json_index) noexcept; - inline size_t after_element() const noexcept; - simdjson_inline tape_type tape_ref_type() const noexcept; - simdjson_inline uint64_t tape_value() const noexcept; - simdjson_inline bool is_double() const noexcept; - simdjson_inline bool is_int64() const noexcept; - simdjson_inline bool is_uint64() const noexcept; - simdjson_inline bool is_false() const noexcept; - simdjson_inline bool is_true() const noexcept; - simdjson_inline bool is_null_on_tape() const noexcept;// different name to avoid clash with is_null. - simdjson_inline uint32_t matching_brace_index() const noexcept; - simdjson_inline uint32_t scope_count() const noexcept; - template - simdjson_inline T next_tape_value() const noexcept; - simdjson_inline uint32_t get_string_length() const noexcept; - simdjson_inline const char * get_c_str() const noexcept; - inline std::string_view get_string_view() const noexcept; - simdjson_inline bool is_document_root() const noexcept; - simdjson_inline bool usable() const noexcept; - - /** The document this element references. */ - const dom::document *doc; - - /** The index of this element on `doc.tape[]` */ - size_t json_index; -}; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_TAPE_REF_H -/* end file simdjson/internal/tape_ref.h */ - -namespace simdjson { -namespace dom { - -/** - * JSON array. - */ -class array { -public: - /** Create a new, invalid array */ - simdjson_inline array() noexcept; - - class iterator { - public: - using value_type = element; - using difference_type = std::ptrdiff_t; - using pointer = void; - using reference = value_type; - using iterator_category = std::forward_iterator_tag; - - /** - * Get the actual value - */ - inline reference operator*() const noexcept; - /** - * Get the next value. - * - * Part of the std::iterator interface. - */ - inline iterator& operator++() noexcept; - /** - * Get the next value. - * - * Part of the std::iterator interface. - */ - inline iterator operator++(int) noexcept; - /** - * Check if these values come from the same place in the JSON. - * - * Part of the std::iterator interface. - */ - inline bool operator!=(const iterator& other) const noexcept; - inline bool operator==(const iterator& other) const noexcept; - - inline bool operator<(const iterator& other) const noexcept; - inline bool operator<=(const iterator& other) const noexcept; - inline bool operator>=(const iterator& other) const noexcept; - inline bool operator>(const iterator& other) const noexcept; - - iterator() noexcept = default; - iterator(const iterator&) noexcept = default; - iterator& operator=(const iterator&) noexcept = default; - private: - simdjson_inline iterator(const internal::tape_ref &tape) noexcept; - internal::tape_ref tape{}; - friend class array; - }; - - /** - * Return the first array element. - * - * Part of the std::iterable interface. - */ - inline iterator begin() const noexcept; - /** - * One past the last array element. - * - * Part of the std::iterable interface. - */ - inline iterator end() const noexcept; - /** - * Get the size of the array (number of immediate children). - * It is a saturated value with a maximum of 0xFFFFFF: if the value - * is 0xFFFFFF then the size is 0xFFFFFF or greater. - */ - inline size_t size() const noexcept; - /** - * Get the total number of slots used by this array on the tape. - * - * Note that this is not the same thing as `size()`, which reports the - * number of actual elements within an array (not counting its children). - * - * Since an element can use 1 or 2 slots on the tape, you can only use this - * to figure out the total size of an array (including its children, - * recursively) if you know its structure ahead of time. - **/ - inline size_t number_of_slots() const noexcept; - /** - * Get the value associated with the given JSON pointer. We use the RFC 6901 - * https://tools.ietf.org/html/rfc6901 standard, interpreting the current node - * as the root of its own JSON document. - * - * dom::parser parser; - * array a = parser.parse(R"([ { "foo": { "a": [ 10, 20, 30 ] }} ])"_padded); - * a.at_pointer("/0/foo/a/1") == 20 - * a.at_pointer("0")["foo"]["a"].at(1) == 20 - * - * @return The value associated with the given JSON pointer, or: - * - NO_SUCH_FIELD if a field does not exist in an object - * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length - * - INCORRECT_TYPE if a non-integer is used to access an array - * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed - */ - inline simdjson_result at_pointer(std::string_view json_pointer) const noexcept; - - /** - * Recursive function which processes the JSON path of each child element - */ - inline void process_json_path_of_child_elements(std::vector::iterator& current, std::vector::iterator& end, const std::string_view& path_suffix, std::vector& accumulator) const noexcept; - - - /** - * Adds support for JSONPath expression with wildcards '*' - */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) const noexcept; - - /** - * Get the value associated with the given JSONPath expression. We only support - * JSONPath queries that trivially convertible to JSON Pointer queries: key - * names and array indices. - * - * https://www.rfc-editor.org/rfc/rfc9535 (RFC 9535) - * - * @return The value associated with the given JSONPath expression, or: - * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails - * - NO_SUCH_FIELD if a field does not exist in an object - * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length - * - INCORRECT_TYPE if a non-integer is used to access an array - */ - inline simdjson_result at_path(std::string_view json_path) const noexcept; - - /** - * Get the value at the given index. This function has linear-time complexity and - * is equivalent to the following: - * - * size_t i=0; - * for (auto element : *this) { - * if (i == index) { return element; } - * i++; - * } - * return INDEX_OUT_OF_BOUNDS; - * - * Avoid calling the at() function repeatedly. - * - * @return The value at the given index, or: - * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length - */ - inline simdjson_result at(size_t index) const noexcept; - - /** - * Gets the values of items in an array element - * This function has linear-time complexity: the values are checked one by one. - * - * @return The child elements of an array - */ - - inline std::vector& get_values(std::vector& out) const noexcept; - - /** - * Implicitly convert object to element - */ - inline operator element() const noexcept; - -private: - simdjson_inline array(const internal::tape_ref &tape) noexcept; - internal::tape_ref tape{}; - friend class element; - friend struct simdjson_result; - template - friend class simdjson::internal::string_builder; -}; - - -} // namespace dom - -/** The result of a JSON conversion that may fail. */ -template<> -struct simdjson_result : public internal::simdjson_result_base { -public: - simdjson_inline simdjson_result() noexcept; ///< @private - simdjson_inline simdjson_result(dom::array value) noexcept; ///< @private - simdjson_inline simdjson_result(error_code error) noexcept; ///< @private - - inline simdjson_result at_pointer(std::string_view json_pointer) const noexcept; - inline void process_json_path_of_child_elements(std::vector::iterator& current, std::vector::iterator& end, const std::string_view& path_suffix, std::vector& accumulator) const noexcept; - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) const noexcept; - inline simdjson_result at_path(std::string_view json_path) const noexcept; - inline simdjson_result at(size_t index) const noexcept; - inline std::vector& get_values(std::vector& out) const noexcept; - -#if SIMDJSON_EXCEPTIONS - inline dom::array::iterator begin() const noexcept(false); - inline dom::array::iterator end() const noexcept(false); - inline size_t size() const noexcept(false); -#endif // SIMDJSON_EXCEPTIONS -}; - - - -} // namespace simdjson - -#if SIMDJSON_SUPPORTS_RANGES -namespace std { -namespace ranges { -template<> -inline constexpr bool enable_view = true; -#if SIMDJSON_EXCEPTIONS -template<> -inline constexpr bool enable_view> = true; -#endif // SIMDJSON_EXCEPTIONS -} // namespace ranges -} // namespace std -#endif // SIMDJSON_SUPPORTS_RANGES - -#endif // SIMDJSON_DOM_ARRAY_H -/* end file simdjson/dom/array.h */ -/* including simdjson/dom/document_stream.h: #include "simdjson/dom/document_stream.h" */ -/* begin file simdjson/dom/document_stream.h */ -#ifndef SIMDJSON_DOCUMENT_STREAM_H -#define SIMDJSON_DOCUMENT_STREAM_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* including simdjson/dom/parser.h: #include "simdjson/dom/parser.h" */ -/* begin file simdjson/dom/parser.h */ -#ifndef SIMDJSON_DOM_PARSER_H -#define SIMDJSON_DOM_PARSER_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* including simdjson/dom/document.h: #include "simdjson/dom/document.h" */ -/* begin file simdjson/dom/document.h */ -#ifndef SIMDJSON_DOM_DOCUMENT_H -#define SIMDJSON_DOM_DOCUMENT_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ - -#include - -namespace simdjson { -namespace dom { - -/** - * A parsed JSON document. - * - * This class cannot be copied, only moved, to avoid unintended allocations. - */ -class document { -public: - /** - * Create a document container with zero capacity. - * - * The parser will allocate capacity as needed. - */ - document() noexcept = default; - ~document() noexcept = default; - - /** - * Take another document's buffers. - * - * @param other The document to take. Its capacity is zeroed and it is invalidated. - */ - document(document &&other) noexcept = default; - /** @private */ - document(const document &) = delete; // Disallow copying - /** - * Take another document's buffers. - * - * @param other The document to take. Its capacity is zeroed. - */ - document &operator=(document &&other) noexcept = default; - /** @private */ - document &operator=(const document &) = delete; // Disallow copying - - /** - * Get the root element of this document as a JSON array. - */ - element root() const noexcept; - - /** - * @private Dump the raw tape for debugging. - * - * @param os the stream to output to. - * @return false if the tape is likely wrong (e.g., you did not parse a valid JSON). - */ - bool dump_raw_tape(std::ostream &os) const noexcept; - - /** @private Structural values. */ - std::unique_ptr tape{}; - - /** @private String values. - * - * Should be at least byte_capacity. - */ - std::unique_ptr string_buf{}; - /** @private Allocate memory to support - * input JSON documents of up to len bytes. - * - * When calling this function, you lose - * all the data. - * - * The memory allocation is strict: you - * can you use this function to increase - * or lower the amount of allocated memory. - * Passing zero clears the memory. - */ - error_code allocate(size_t len) noexcept; - /** @private Capacity in bytes, in terms - * of how many bytes of input JSON we can - * support. - */ - size_t capacity() const noexcept; - - -private: - size_t allocated_capacity{0}; - friend class parser; -}; // class document - -} // namespace dom -} // namespace simdjson - -#endif // SIMDJSON_DOM_DOCUMENT_H -/* end file simdjson/dom/document.h */ - -namespace simdjson { - -namespace dom { - -/** - * A persistent document parser. - * - * The parser is designed to be reused, holding the internal buffers necessary to do parsing, - * as well as memory for a single document. The parsed document is overwritten on each parse. - * - * This class cannot be copied, only moved, to avoid unintended allocations. - * - * @note Moving a parser instance may invalidate "dom::element" instances. If you need to - * preserve both the "dom::element" instances and the parser, consider wrapping the parser - * instance in a std::unique_ptr instance: - * - * std::unique_ptr parser(new dom::parser{}); - * auto error = parser->load(f).get(root); - * - * You can then move std::unique_ptr safely. - * - * @note This is not thread safe: one parser cannot produce two documents at the same time! - */ -class parser { -public: - /** - * Create a JSON parser. - * - * The new parser will have zero capacity. - * - * @param max_capacity The maximum document length the parser can automatically handle. The parser - * will allocate more capacity on an as needed basis (when it sees documents too big to handle) - * up to this amount. The parser still starts with zero capacity no matter what this number is: - * to allocate an initial capacity, call allocate() after constructing the parser. - * Defaults to SIMDJSON_MAXSIZE_BYTES (the largest single document simdjson can process). - */ - simdjson_inline explicit parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES) noexcept; - /** - * Take another parser's buffers and state. - * - * @param other The parser to take. Its capacity is zeroed. - */ - simdjson_inline parser(parser &&other) noexcept; - parser(const parser &) = delete; ///< @private Disallow copying - /** - * Take another parser's buffers and state. - * - * @param other The parser to take. Its capacity is zeroed. - */ - simdjson_inline parser &operator=(parser &&other) noexcept; - parser &operator=(const parser &) = delete; ///< @private Disallow copying - - /** Deallocate the JSON parser. */ - ~parser()=default; - - /** - * Load a JSON document from a file and return a reference to it. - * - * dom::parser parser; - * const element doc = parser.load("jsonexamples/twitter.json"); - * - * The function is eager: the file's content is loaded in memory inside the parser instance - * and immediately parsed. The file can be deleted after the `parser.load` call. - * - * ### IMPORTANT: Document Lifetime - * - * The JSON document still lives in the parser: this is the most efficient way to parse JSON - * documents because it reuses the same buffers, but you *must* use the document before you - * destroy the parser or call parse() again. - * - * Moving the parser instance is safe, but it invalidates the element instances. You may store - * the parser instance without moving it by wrapping it inside an `unique_ptr` instance like - * so: `std::unique_ptr parser(new dom::parser{});`. - * - * ### Parser Capacity - * - * If the parser's current capacity is less than the file length, it will allocate enough capacity - * to handle it (up to max_capacity). - * - * ## Windows and Unicode - * - * Windows users who need to read files with non-ANSI characters in the - * name should set their code page to UTF-8 (65001) before calling this - * function. This should be the default with Windows 11 and better. - * Further, they may use the AreFileApisANSI function to determine whether - * the filename is interpreted using the ANSI or the system default OEM - * codepage, and they may call SetFileApisToOEM accordingly. - * - * @param path The path to load. - * @return The document, or an error: - * - IO_ERROR if there was an error opening or reading the file. - * Be mindful that on some 32-bit systems, - * the file size might be limited to 2 GB. - * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. - * - CAPACITY if the parser does not have enough capacity and len > max_capacity. - * - other json errors if parsing fails. You should not rely on these errors to always the same for the - * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). - */ - inline simdjson_result load(std::string_view path) & noexcept; - inline simdjson_result load(std::string_view path) && = delete ; - - /** - * Load a JSON document from a file into a provide document instance and return a temporary reference to it. - * It is similar to the function `load` except that instead of parsing into the internal - * `document` instance associated with the parser, it allows the user to provide a document - * instance. - * - * dom::parser parser; - * dom::document doc; - * element doc_root = parser.load_into_document(doc, "jsonexamples/twitter.json"); - * - * The function is eager: the file's content is loaded in memory inside the parser instance - * and immediately parsed. The file can be deleted after the `parser.load_into_document` call. - * - * ### IMPORTANT: Document Lifetime - * - * After the call to load_into_document, the parser is no longer needed. - * - * The JSON document lives in the document instance: you must keep the document - * instance alive while you navigate through it (i.e., used the returned value from - * load_into_document). You are encourage to reuse the document instance - * many times with new data to avoid reallocations: - * - * dom::document doc; - * element doc_root1 = parser.load_into_document(doc, "jsonexamples/twitter.json"); - * //... doc_root1 is a pointer inside doc - * element doc_root2 = parser.load_into_document(doc, "jsonexamples/twitter.json"); - * //... doc_root2 is a pointer inside doc - * // at this point doc_root1 is no longer safe - * - * Moving the document instance is safe, but it invalidates the element instances. After - * moving a document, you can recover safe access to the document root with its `root()` method. - * - * @param doc The document instance where the parsed data will be stored (on success). - * @param path The path to load. - * @return The document, or an error: - * - IO_ERROR if there was an error opening or reading the file. - * Be mindful that on some 32-bit systems, - * the file size might be limited to 2 GB. - * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. - * - CAPACITY if the parser does not have enough capacity and len > max_capacity. - * - other json errors if parsing fails. You should not rely on these errors to always the same for the - * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). - */ - inline simdjson_result load_into_document(document& doc, std::string_view path) & noexcept; - inline simdjson_result load_into_document(document& doc, std::string_view path) && =delete; - - /** - * Parse a JSON document and return a temporary reference to it. - * - * dom::parser parser; - * element doc_root = parser.parse(buf, len); - * - * The function eagerly parses the input: the input can be modified and discarded after - * the `parser.parse(buf, len)` call has completed. - * - * ### IMPORTANT: Document Lifetime - * - * The JSON document still lives in the parser: this is the most efficient way to parse JSON - * documents because it reuses the same buffers, but you *must* use the document before you - * destroy the parser or call parse() again. - * - * Moving the parser instance is safe, but it invalidates the element instances. You may store - * the parser instance without moving it by wrapping it inside an `unique_ptr` instance like - * so: `std::unique_ptr parser(new dom::parser{});`. - * - * ### REQUIRED: Buffer Padding - * - * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what - * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you - * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the - * SIMDJSON_PADDING bytes to avoid runtime warnings. - * - * If realloc_if_needed is true (the default), it is assumed that the buffer does *not* have enough padding, - * and it is copied into an enlarged temporary buffer before parsing. Thus the following is safe: - * - * const char *json = R"({"key":"value"})"; - * const size_t json_len = std::strlen(json); - * simdjson::dom::parser parser; - * simdjson::dom::element element = parser.parse(json, json_len); - * - * If you set realloc_if_needed to false (e.g., parser.parse(json, json_len, false)), - * you must provide a buffer with at least SIMDJSON_PADDING extra bytes at the end. - * The benefit of setting realloc_if_needed to false is that you avoid a temporary - * memory allocation and a copy. - * - * The padded bytes may be read. It is not important how you initialize - * these bytes though we recommend a sensible default like null character values or spaces. - * For example, the following low-level code is safe: - * - * const char *json = R"({"key":"value"})"; - * const size_t json_len = std::strlen(json); - * std::unique_ptr padded_json_copy{new char[json_len + SIMDJSON_PADDING]}; - * std::memcpy(padded_json_copy.get(), json, json_len); - * std::memset(padded_json_copy.get() + json_len, '\0', SIMDJSON_PADDING); - * simdjson::dom::parser parser; - * simdjson::dom::element element = parser.parse(padded_json_copy.get(), json_len, false); - * - * ### std::string references - * - * Whenever you pass an std::string reference, the parser may access the bytes beyond the end of - * the string but before the end of the allocated memory (std::string::capacity()). - * If you are using a sanitizer that checks for reading uninitialized bytes or std::string's - * container-overflow checks, you may encounter sanitizer warnings. - * You can safely ignore these warnings. Or you can call simdjson::pad(std::string&) to pad the - * string with SIMDJSON_PADDING spaces: this function returns a simdjson::padding_string_view - * which can be be passed to the parser's parse function: - * - * std::string json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )"; - * element doc = parser.parse(simdjson::pad(json)); - * - * ### Parser Capacity - * - * If the parser's current capacity is less than len, it will allocate enough capacity - * to handle it (up to max_capacity). - * - * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless - * realloc_if_needed is true. - * @param len The length of the JSON. - * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. - * @return An element pointing at the root of the document, or an error: - * - MEMALLOC if realloc_if_needed is true or the parser does not have enough capacity, - * and memory allocation fails. - * - CAPACITY if the parser does not have enough capacity and len > max_capacity. - * - other json errors if parsing fails. You should not rely on these errors to always the same for the - * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). - */ - inline simdjson_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) & noexcept; - inline simdjson_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) && =delete; - /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_inline simdjson_result parse(const char *buf, size_t len, bool realloc_if_needed = true) & noexcept; - simdjson_inline simdjson_result parse(const char *buf, size_t len, bool realloc_if_needed = true) && =delete; - /** @overload parse(const std::string &) */ - simdjson_inline simdjson_result parse(const std::string &s) & noexcept; - simdjson_inline simdjson_result parse(const std::string &s) && =delete; - /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_inline simdjson_result parse(const padded_string &s) & noexcept; - simdjson_inline simdjson_result parse(const padded_string &s) && =delete; - /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_inline simdjson_result parse(const padded_string_view &v) & noexcept; - simdjson_inline simdjson_result parse(const padded_string_view &v) && =delete; - - /** @private We do not want to allow implicit conversion from C string to std::string. */ - simdjson_inline simdjson_result parse(const char *buf) noexcept = delete; - - /** - * Parse a JSON document into a provide document instance and return a temporary reference to it. - * It is similar to the function `parse` except that instead of parsing into the internal - * `document` instance associated with the parser, it allows the user to provide a document - * instance. - * - * dom::parser parser; - * dom::document doc; - * element doc_root = parser.parse_into_document(doc, buf, len); - * - * The function eagerly parses the input: the input can be modified and discarded after - * the `parser.parse(buf, len)` call has completed. - * - * ### IMPORTANT: Document Lifetime - * - * After the call to parse_into_document, the parser is no longer needed. - * - * The JSON document lives in the document instance: you must keep the document - * instance alive while you navigate through it (i.e., used the returned value from - * parse_into_document). You are encourage to reuse the document instance - * many times with new data to avoid reallocations: - * - * dom::document doc; - * element doc_root1 = parser.parse_into_document(doc, buf1, len); - * //... doc_root1 is a pointer inside doc - * element doc_root2 = parser.parse_into_document(doc, buf1, len); - * //... doc_root2 is a pointer inside doc - * // at this point doc_root1 is no longer safe - * - * Moving the document instance is safe, but it invalidates the element instances. After - * moving a document, you can recover safe access to the document root with its `root()` method. - * - * @param doc The document instance where the parsed data will be stored (on success). - * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless - * realloc_if_needed is true. - * @param len The length of the JSON. - * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. - * @return An element pointing at the root of document, or an error: - * - MEMALLOC if realloc_if_needed is true or the parser does not have enough capacity, - * and memory allocation fails. - * - CAPACITY if the parser does not have enough capacity and len > max_capacity. - * - other json errors if parsing fails. You should not rely on these errors to always the same for the - * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). - */ - inline simdjson_result parse_into_document(document& doc, const uint8_t *buf, size_t len, bool realloc_if_needed = true) & noexcept; - inline simdjson_result parse_into_document(document& doc, const uint8_t *buf, size_t len, bool realloc_if_needed = true) && =delete; - /** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_inline simdjson_result parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = true) & noexcept; - simdjson_inline simdjson_result parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = true) && =delete; - /** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_inline simdjson_result parse_into_document(document& doc, const std::string &s) & noexcept; - simdjson_inline simdjson_result parse_into_document(document& doc, const std::string &s) && =delete; - /** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_inline simdjson_result parse_into_document(document& doc, const padded_string &s) & noexcept; - simdjson_inline simdjson_result parse_into_document(document& doc, const padded_string &s) && =delete; - - /** @private We do not want to allow implicit conversion from C string to std::string. */ - simdjson_inline simdjson_result parse_into_document(document& doc, const char *buf) noexcept = delete; - - /** - * Load a file containing many JSON documents. - * - * dom::parser parser; - * for (const element doc : parser.load_many(path)) { - * cout << std::string(doc["title"]) << endl; - * } - * - * The file is loaded in memory and can be safely deleted after the `parser.load_many(path)` - * function has returned. The memory is held by the `parser` instance. - * - * The function is lazy: it may be that no more than one JSON document at a time is parsed. - * And, possibly, no document many have been parsed when the `parser.load_many(path)` function - * returned. - * - * If there is a UTF-8 BOM, the parser skips it. - * - * ### Format - * - * The file must contain a series of one or more JSON documents, concatenated into a single - * buffer, separated by whitespace. It effectively parses until it has a fully valid document, - * then starts parsing the next document at that point. (It does this with more parallelism and - * lookahead than you might think, though.) - * - * Documents that consist of an object or array may omit the whitespace between them, concatenating - * with no separator. documents that consist of a single primitive (i.e. documents that are not - * arrays or objects) MUST be separated with whitespace. - * - * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse. - * Setting batch_size to excessively large or excessively small values may impact negatively the - * performance. - * - * ### Error Handling - * - * All errors are returned during iteration: if there is a global error such as memory allocation, - * it will be yielded as the first result. Iteration always stops after the first error. - * - * As with all other simdjson methods, non-exception error handling is readily available through - * the same interface, requiring you to check the error before using the document: - * - * dom::parser parser; - * dom::document_stream docs; - * auto error = parser.load_many(path).get(docs); - * if (error) { cerr << error << endl; exit(1); } - * for (auto doc : docs) { - * std::string_view title; - * if ((error = doc["title"].get(title)) { cerr << error << endl; exit(1); } - * cout << title << endl; - * } - * - * ### Threads - * - * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the - * hood to do some lookahead. - * - * ### Parser Capacity - * - * If the parser's current capacity is less than batch_size, it will allocate enough capacity - * to handle it (up to max_capacity). - * - * @param path File name pointing at the concatenated JSON to parse. - * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet - * spot is cache-related: small enough to fit in cache, yet big enough to - * parse as many documents as possible in one tight loop. - * Defaults to 1MB (as simdjson::dom::DEFAULT_BATCH_SIZE), which has been a reasonable sweet - * spot in our tests. - * If you set the batch_size to a value smaller than simdjson::dom::MINIMAL_BATCH_SIZE - * (currently 32B), it will be replaced by simdjson::dom::MINIMAL_BATCH_SIZE. - * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: - * - IO_ERROR if there was an error opening or reading the file. - * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. - * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. - * - other json errors if parsing fails. You should not rely on these errors to always the same for the - * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). - */ - inline simdjson_result load_many(std::string_view path, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; - - /** - * Parse a buffer containing many JSON documents. - * - * dom::parser parser; - * for (element doc : parser.parse_many(buf, len)) { - * cout << std::string(doc["title"]) << endl; - * } - * - * No copy of the input buffer is made. - * - * The function is lazy: it may be that no more than one JSON document at a time is parsed. - * And, possibly, no document many have been parsed when the `parser.load_many(path)` function - * returned. - * - * The caller is responsabile to ensure that the input string data remains unchanged and is - * not deleted during the loop. In particular, the following is unsafe and will not compile: - * - * auto docs = parser.parse_many("[\"temporary data\"]"_padded); - * // here the string "[\"temporary data\"]" may no longer exist in memory - * // the parser instance may not have even accessed the input yet - * for (element doc : docs) { - * cout << std::string(doc["title"]) << endl; - * } - * - * The following is safe: - * - * auto json = "[\"temporary data\"]"_padded; - * auto docs = parser.parse_many(json); - * for (element doc : docs) { - * cout << std::string(doc["title"]) << endl; - * } - * - * If there is a UTF-8 BOM, the parser skips it. - * - * ### Format - * - * The buffer must contain a series of one or more JSON documents, concatenated into a single - * buffer, separated by whitespace. It effectively parses until it has a fully valid document, - * then starts parsing the next document at that point. (It does this with more parallelism and - * lookahead than you might think, though.) - * - * documents that consist of an object or array may omit the whitespace between them, concatenating - * with no separator. documents that consist of a single primitive (i.e. documents that are not - * arrays or objects) MUST be separated with whitespace. - * - * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse. - * Setting batch_size to excessively large or excessively small values may impact negatively the - * performance. - * - * ### Error Handling - * - * All errors are returned during iteration: if there is a global error such as memory allocation, - * it will be yielded as the first result. Iteration always stops after the first error. - * - * As with all other simdjson methods, non-exception error handling is readily available through - * the same interface, requiring you to check the error before using the document: - * - * dom::parser parser; - * dom::document_stream docs; - * auto error = parser.load_many(path).get(docs); - * if (error) { cerr << error << endl; exit(1); } - * for (auto doc : docs) { - * std::string_view title; - * if ((error = doc["title"].get(title)) { cerr << error << endl; exit(1); } - * cout << title << endl; - * } - * - * ### REQUIRED: Buffer Padding - * - * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what - * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you - * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the - * SIMDJSON_PADDING bytes to avoid runtime warnings. - * - * ### Threads - * - * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the - * hood to do some lookahead. - * - * ### Parser Capacity - * - * If the parser's current capacity is less than batch_size, it will allocate enough capacity - * to handle it (up to max_capacity). - * - * @param buf The concatenated JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes. - * @param len The length of the concatenated JSON. - * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet - * spot is cache-related: small enough to fit in cache, yet big enough to - * parse as many documents as possible in one tight loop. - * Defaults to 10MB, which has been a reasonable sweet spot in our tests. - * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: - * - MEMALLOC if the parser does not have enough capacity and memory allocation fails - * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. - * - other json errors if parsing fails. You should not rely on these errors to always the same for the - * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). - */ - inline simdjson_result parse_many(const uint8_t *buf, size_t len, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; - /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline simdjson_result parse_many(const char *buf, size_t len, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; - /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline simdjson_result parse_many(const std::string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; - inline simdjson_result parse_many(const std::string &&s, size_t batch_size) = delete;// unsafe - /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline simdjson_result parse_many(const padded_string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; - inline simdjson_result parse_many(const padded_string &&s, size_t batch_size) = delete;// unsafe - - /** @private We do not want to allow implicit conversion from C string to std::string. */ - simdjson_result parse_many(const char *buf, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept = delete; - - /** - * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length - * and `max_depth` depth. - * - * @param capacity The new capacity. - * @param max_depth The new max_depth. Defaults to DEFAULT_MAX_DEPTH. - * @return The error, if there is one. - */ - simdjson_warn_unused inline error_code allocate(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept; - -#ifndef SIMDJSON_DISABLE_DEPRECATED_API - /** - * @private deprecated because it returns bool instead of error_code, which is our standard for - * failures. Use allocate() instead. - * - * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length - * and `max_depth` depth. - * - * @param capacity The new capacity. - * @param max_depth The new max_depth. Defaults to DEFAULT_MAX_DEPTH. - * @return true if successful, false if allocation failed. - */ - [[deprecated("Use allocate() instead.")]] - simdjson_warn_unused inline bool allocate_capacity(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept; -#endif // SIMDJSON_DISABLE_DEPRECATED_API - /** - * The largest document this parser can support without reallocating. - * - * @return Current capacity, in bytes. - */ - simdjson_inline size_t capacity() const noexcept; - - /** - * The largest document this parser can automatically support. - * - * The parser may reallocate internal buffers as needed up to this amount. - * - * @return Maximum capacity, in bytes. - */ - simdjson_inline size_t max_capacity() const noexcept; - - /** - * The maximum level of nested object and arrays supported by this parser. - * - * @return Maximum depth, in bytes. - */ - simdjson_pure simdjson_inline size_t max_depth() const noexcept; - - /** - * Set max_capacity. This is the largest document this parser can automatically support. - * - * The parser may reallocate internal buffers as needed up to this amount as documents are passed - * to it. - * - * Note: To avoid limiting the memory to an absurd value, such as zero or two bytes, - * iff you try to set max_capacity to a value lower than MINIMAL_DOCUMENT_CAPACITY, - * then the maximal capacity is set to MINIMAL_DOCUMENT_CAPACITY. - * - * This call will not allocate or deallocate, even if capacity is currently above max_capacity. - * - * @param max_capacity The new maximum capacity, in bytes. - */ - simdjson_inline void set_max_capacity(size_t max_capacity) noexcept; - -#ifdef SIMDJSON_THREADS_ENABLED - /** - * The parser instance can use threads when they are available to speed up some - * operations. It is enabled by default. Changing this attribute will change the - * behavior of the parser for future operations. Set to true by default. - */ - bool threaded{true}; -#else - /** - * When SIMDJSON_THREADS_ENABLED is not defined, the parser instance cannot use threads. - */ - bool threaded{false}; -#endif - /** @private Use the new DOM API instead */ - class Iterator; - /** @private Use simdjson_error instead */ - using InvalidJSON [[deprecated("Use simdjson_error instead")]] = simdjson_error; - - /** @private [for benchmarking access] The implementation to use */ - std::unique_ptr implementation{}; - - /** @private Use `if (parser.parse(...).error())` instead */ - bool valid{false}; - /** @private Use `parser.parse(...).error()` instead */ - error_code error{UNINITIALIZED}; - - /** @private Use `parser.parse(...).value()` instead */ - document doc{}; - - /** @private returns true if the document parsed was valid */ - [[deprecated("Use the result of parser.parse() instead")]] - inline bool is_valid() const noexcept; - - /** - * @private return an error code corresponding to the last parsing attempt, see - * simdjson.h will return UNINITIALIZED if no parsing was attempted - */ - [[deprecated("Use the result of parser.parse() instead")]] - inline int get_error_code() const noexcept; - - /** @private return the string equivalent of "get_error_code" */ - [[deprecated("Use error_message() on the result of parser.parse() instead, or cout << error")]] - inline std::string get_error_message() const noexcept; - - /** @private */ - [[deprecated("Use cout << on the result of parser.parse() instead")]] - inline bool print_json(std::ostream &os) const noexcept; - - /** @private Private and deprecated: use `parser.parse(...).doc.dump_raw_tape()` instead */ - inline bool dump_raw_tape(std::ostream &os) const noexcept; - - - /** - * When enabled, big integers (exceeding uint64 range) are stored as strings - * in the tape instead of returning BIGINT_ERROR. Default: false. - */ - inline void number_as_string(bool enabled) noexcept { _number_as_string = enabled; } - inline bool number_as_string() const noexcept { return _number_as_string; } - -private: - /** - * The maximum document length this parser will automatically support. - * - * The parser will not be automatically allocated above this amount. - */ - size_t _max_capacity; - - /** Whether to store big integers as strings instead of returning BIGINT_ERROR */ - bool _number_as_string{false}; - - /** - * The loaded buffer (reused each time load() is called) - */ - std::unique_ptr loaded_bytes; - - /** Capacity of loaded_bytes buffer. */ - size_t _loaded_bytes_capacity{0}; - - // all nodes are stored on the doc.tape using a 64-bit word. - // - // strings, double and ints are stored as - // a 64-bit word with a pointer to the actual value - // - // - // - // for objects or arrays, store [ or { at the beginning and } and ] at the - // end. For the openings ([ or {), we annotate them with a reference to the - // location on the doc.tape of the end, and for then closings (} and ]), we - // annotate them with a reference to the location of the opening - // - // - - /** - * Ensure we have enough capacity to handle at least desired_capacity bytes, - * and auto-allocate if not. This also allocates memory if needed in the - * internal document. - */ - inline error_code ensure_capacity(size_t desired_capacity) noexcept; - /** - * Ensure we have enough capacity to handle at least desired_capacity bytes, - * and auto-allocate if not. This also allocates memory if needed in the - * provided document. - */ - inline error_code ensure_capacity(document& doc, size_t desired_capacity) noexcept; - - /** Read the file into loaded_bytes */ - inline simdjson_result read_file(std::string_view path) noexcept; - - friend class parser::Iterator; - friend class document_stream; - - -}; // class parser - -} // namespace dom -} // namespace simdjson - -#endif // SIMDJSON_DOM_PARSER_H -/* end file simdjson/dom/parser.h */ - -#ifdef SIMDJSON_THREADS_ENABLED -#include -#include -#include -#endif - -namespace simdjson { -namespace dom { - -#ifdef SIMDJSON_THREADS_ENABLED -/** @private Custom worker class **/ -struct stage1_worker { - stage1_worker() noexcept = default; - stage1_worker(const stage1_worker&) = delete; - stage1_worker(stage1_worker&&) = delete; - stage1_worker operator=(const stage1_worker&) = delete; - ~stage1_worker(); - /** - * We only start the thread when it is needed, not at object construction, this may throw. - * You should only call this once. - **/ - void start_thread(); - /** - * Start a stage 1 job. You should first call 'run', then 'finish'. - * You must call start_thread once before. - */ - void run(document_stream * ds, dom::parser * stage1, size_t next_batch_start); - /** Wait for the run to finish (blocking). You should first call 'run', then 'finish'. **/ - void finish(); - -private: - - /** - * Normally, we would never stop the thread. But we do in the destructor. - * This function is only safe assuming that you are not waiting for results. You - * should have called run, then finish, and be done. - **/ - void stop_thread(); - - std::thread thread{}; - /** These three variables define the work done by the thread. **/ - dom::parser * stage1_thread_parser{}; - size_t _next_batch_start{}; - document_stream * owner{}; - /** - * We have two state variables. This could be streamlined to one variable in the future but - * we use two for clarity. - */ - bool has_work{false}; - bool can_work{true}; - - /** - * We lock using a mutex. - */ - std::mutex locking_mutex{}; - std::condition_variable cond_var{}; -}; -#endif - -/** - * A forward-only stream of documents. - * - * Produced by parser::parse_many. - * - */ -class document_stream { -public: - /** - * Construct an uninitialized document_stream. - * - * ```cpp - * document_stream docs; - * error = parser.parse_many(json).get(docs); - * ``` - */ - simdjson_inline document_stream() noexcept; - /** Move one document_stream to another. */ - simdjson_inline document_stream(document_stream &&other) noexcept = default; - /** Move one document_stream to another. */ - simdjson_inline document_stream &operator=(document_stream &&other) noexcept = default; - - simdjson_inline ~document_stream() noexcept; - /** - * Returns the input size in bytes. - */ - inline size_t size_in_bytes() const noexcept; - /** - * After iterating through the stream, this method - * returns the number of bytes that were not parsed at the end - * of the stream. If truncated_bytes() differs from zero, - * then the input was truncated maybe because incomplete JSON - * documents were found at the end of the stream. You - * may need to process the bytes in the interval [size_in_bytes()-truncated_bytes(), size_in_bytes()). - * - * You should only call truncated_bytes() after streaming through all - * documents, like so: - * - * document_stream stream = parser.parse_many(json,window); - * for(auto doc : stream) { - * // do something with doc - * } - * size_t truncated = stream.truncated_bytes(); - * - */ - inline size_t truncated_bytes() const noexcept; - /** - * An iterator through a forward-only stream of documents. - */ - class iterator { - public: - using value_type = simdjson_result; - using reference = value_type; - - using difference_type = std::ptrdiff_t; - - using iterator_category = std::input_iterator_tag; - - /** - * Default constructor. - */ - simdjson_inline iterator() noexcept; - /** - * Get the current document (or error). - */ - simdjson_inline reference operator*() noexcept; - /** - * Advance to the next document (prefix). - */ - inline iterator& operator++() noexcept; - /** - * Check if we're at the end yet. - * @param other the end iterator to compare to. - */ - simdjson_inline bool operator!=(const iterator &other) const noexcept; - /** - * @private - * - * Gives the current index in the input document in bytes. - * - * document_stream stream = parser.parse_many(json,window); - * for(auto i = stream.begin(); i != stream.end(); ++i) { - * auto doc = *i; - * size_t index = i.current_index(); - * } - * - * This function (current_index()) is experimental and the usage - * may change in future versions of simdjson: we find the API somewhat - * awkward and we would like to offer something friendlier. - */ - simdjson_inline size_t current_index() const noexcept; - /** - * @private - * - * Gives a view of the current document. - * - * document_stream stream = parser.parse_many(json,window); - * for(auto i = stream.begin(); i != stream.end(); ++i) { - * auto doc = *i; - * std::string_view v = i->source(); - * } - * - * The returned string_view instance is simply a map to the (unparsed) - * source string: it may thus include white-space characters and all manner - * of padding. - * - * This function (source()) is experimental and the usage - * may change in future versions of simdjson: we find the API somewhat - * awkward and we would like to offer something friendlier. - */ - simdjson_inline std::string_view source() const noexcept; - - private: - simdjson_inline iterator(document_stream *s, bool finished) noexcept; - /** The document_stream we're iterating through. */ - document_stream* stream; - /** Whether we're finished or not. */ - bool finished; - friend class document_stream; - }; - - /** - * Start iterating the documents in the stream. - */ - simdjson_inline iterator begin() noexcept; - /** - * The end of the stream, for iterator comparison purposes. - */ - simdjson_inline iterator end() noexcept; - -private: - - document_stream &operator=(const document_stream &) = delete; // Disallow copying - document_stream(const document_stream &other) = delete; // Disallow copying - - /** - * Construct a document_stream. Does not allocate or parse anything until the iterator is - * used. - * - * @param parser is a reference to the parser instance used to generate this document_stream - * @param buf is the raw byte buffer we need to process - * @param len is the length of the raw byte buffer in bytes - * @param batch_size is the size of the windows (must be strictly greater or equal to the largest JSON document) - */ - simdjson_inline document_stream( - dom::parser &parser, - const uint8_t *buf, - size_t len, - size_t batch_size - ) noexcept; - - /** - * Parse the first document in the buffer. Used by begin(), to handle allocation and - * initialization. - */ - inline void start() noexcept; - - /** - * Parse the next document found in the buffer previously given to document_stream. - * - * The content should be a valid JSON document encoded as UTF-8. If there is a - * UTF-8 BOM, the parser skips it. - * - * You do NOT need to pre-allocate a parser. This function takes care of - * pre-allocating a capacity defined by the batch_size defined when creating the - * document_stream object. - * - * The function returns simdjson::EMPTY if there is no more data to be parsed. - * - * The function returns simdjson::SUCCESS (as integer = 0) in case of success - * and indicates that the buffer has successfully been parsed to the end. - * Every document it contained has been parsed without error. - * - * The function returns an error code from simdjson/simdjson.h in case of failure - * such as simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth; - * the simdjson::error_message function converts these error codes into a string). - * - * You can also check validity by calling parser.is_valid(). The same parser can - * and should be reused for the other documents in the buffer. - */ - inline void next() noexcept; - - /** - * Pass the next batch through stage 1 and return when finished. - * When threads are enabled, this may wait for the stage 1 thread to finish. - */ - inline void load_batch() noexcept; - - /** Get the next document index. */ - inline size_t next_batch_start() const noexcept; - - /** Pass the next batch through stage 1 with the given parser. */ - inline error_code run_stage1(dom::parser &p, size_t batch_start) noexcept; - - dom::parser *parser; - const uint8_t *buf; - size_t len; - size_t batch_size; - /** The error (or lack thereof) from the current document. */ - error_code error; - size_t batch_start{0}; - size_t doc_index{}; -#ifdef SIMDJSON_THREADS_ENABLED - /** Indicates whether we use threads. Note that this needs to be a constant during the execution of the parsing. */ - bool use_thread; - - inline void load_from_stage1_thread() noexcept; - - /** Start a thread to run stage 1 on the next batch. */ - inline void start_stage1_thread() noexcept; - - /** Wait for the stage 1 thread to finish and capture the results. */ - inline void finish_stage1_thread() noexcept; - - /** The error returned from the stage 1 thread. */ - error_code stage1_thread_error{UNINITIALIZED}; - /** The thread used to run stage 1 against the next batch in the background. */ - friend struct stage1_worker; - std::unique_ptr worker{new(std::nothrow) stage1_worker()}; - /** - * The parser used to run stage 1 in the background. Will be swapped - * with the regular parser when finished. - */ - dom::parser stage1_thread_parser{}; -#endif // SIMDJSON_THREADS_ENABLED - - friend class dom::parser; - friend struct simdjson_result; - friend struct internal::simdjson_result_base; - -}; // class document_stream - -} // namespace dom - -template<> -struct simdjson_result : public internal::simdjson_result_base { -public: - simdjson_inline simdjson_result() noexcept; ///< @private - simdjson_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_inline simdjson_result(dom::document_stream &&value) noexcept; ///< @private - -#if SIMDJSON_EXCEPTIONS - simdjson_inline dom::document_stream::iterator begin() noexcept(false); - simdjson_inline dom::document_stream::iterator end() noexcept(false); -#else // SIMDJSON_EXCEPTIONS -#ifndef SIMDJSON_DISABLE_DEPRECATED_API - [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] - simdjson_inline dom::document_stream::iterator begin() noexcept; - [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] - simdjson_inline dom::document_stream::iterator end() noexcept; -#endif // SIMDJSON_DISABLE_DEPRECATED_API -#endif // SIMDJSON_EXCEPTIONS -}; // struct simdjson_result - -} // namespace simdjson - -#endif // SIMDJSON_DOCUMENT_STREAM_H -/* end file simdjson/dom/document_stream.h */ -/* skipped duplicate #include "simdjson/dom/document.h" */ -/* including simdjson/dom/element.h: #include "simdjson/dom/element.h" */ -/* begin file simdjson/dom/element.h */ -#ifndef SIMDJSON_DOM_ELEMENT_H -#define SIMDJSON_DOM_ELEMENT_H - -#include - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/array.h" */ - -namespace simdjson { -namespace dom { - -/** - * The actual concrete type of a JSON element - * This is the type it is most easily cast to with get<>. - */ -enum class element_type { - ARRAY = '[', ///< dom::array - OBJECT = '{', ///< dom::object - INT64 = 'l', ///< int64_t - UINT64 = 'u', ///< uint64_t: any integer that fits in uint64_t but *not* int64_t - DOUBLE = 'd', ///< double: Any number with a "." or "e" that fits in double. - STRING = '"', ///< std::string_view - BOOL = 't', ///< bool - NULL_VALUE = 'n', ///< null - BIGINT = 'Z' ///< std::string_view: big integer stored as raw digit string -}; - -/** - * A JSON element. - * - * References an element in a JSON document, representing a JSON null, boolean, string, number, - * array or object. - */ -class element { -public: - /** Create a new, invalid element. */ - simdjson_inline element() noexcept; - - /** The type of this element. */ - simdjson_inline element_type type() const noexcept; - - /** - * Cast this element to an array. - * - * @returns An object that can be used to iterate the array, or: - * INCORRECT_TYPE if the JSON element is not an array. - */ - inline simdjson_result get_array() const noexcept; - /** - * Cast this element to an object. - * - * @returns An object that can be used to look up or iterate the object's fields, or: - * INCORRECT_TYPE if the JSON element is not an object. - */ - inline simdjson_result get_object() const noexcept; - /** - * Cast this element to a null-terminated C string. - * - * The string is guaranteed to be valid UTF-8. - * - * The length of the string is given by get_string_length(). Because JSON strings - * may contain null characters, it may be incorrect to use strlen to determine the - * string length. - * - * It is possible to get a single string_view instance which represents both the string - * content and its length: see get_string(). - * - * @returns A pointer to a null-terminated UTF-8 string. This string is stored in the parser and will - * be invalidated the next time it parses a document or when it is destroyed. - * Returns INCORRECT_TYPE if the JSON element is not a string. - */ - inline simdjson_result get_c_str() const noexcept; - /** - * Gives the length in bytes of the string. - * - * It is possible to get a single string_view instance which represents both the string - * content and its length: see get_string(). - * - * @returns A string length in bytes. - * Returns INCORRECT_TYPE if the JSON element is not a string. - */ - inline simdjson_result get_string_length() const noexcept; - /** - * Cast this element to a string. - * - * The string is guaranteed to be valid UTF-8. - * - * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next time it - * parses a document or when it is destroyed. - * Returns INCORRECT_TYPE if the JSON element is not a string. - */ - inline simdjson_result get_string() const noexcept; - /** - * Cast this element to a signed integer. - * - * @returns A signed 64-bit integer. - * Returns INCORRECT_TYPE if the JSON element is not an integer, or NUMBER_OUT_OF_RANGE - * if it is negative. - */ - inline simdjson_result get_int64() const noexcept; - /** - * Cast this element to an unsigned integer. - * - * @returns An unsigned 64-bit integer. - * Returns INCORRECT_TYPE if the JSON element is not an integer, or NUMBER_OUT_OF_RANGE - * if it is too large. - */ - inline simdjson_result get_uint64() const noexcept; - /** - * Cast this element to a double floating-point. - * - * @returns A double value. - * Returns INCORRECT_TYPE if the JSON element is not a number. - */ - inline simdjson_result get_double() const noexcept; - /** - * Cast this element to a bool. - * - * @returns A bool value. - * Returns INCORRECT_TYPE if the JSON element is not a boolean. - */ - inline simdjson_result get_bool() const noexcept; - - /** - * Read this element as a big integer (raw digit string). - * - * @returns A string_view of the raw digits, or: - * INCORRECT_TYPE if the JSON element is not a big integer. - */ - inline simdjson_result get_bigint() const noexcept; - - /** - * Whether this element is a json array. - * - * Equivalent to is(). - */ - inline bool is_array() const noexcept; - /** - * Whether this element is a json object. - * - * Equivalent to is(). - */ - inline bool is_object() const noexcept; - /** - * Whether this element is a json string. - * - * Equivalent to is() or is(). - */ - inline bool is_string() const noexcept; - /** - * Whether this element is a json number that fits in a signed 64-bit integer. - * - * Equivalent to is(). - */ - inline bool is_int64() const noexcept; - /** - * Whether this element is a json number that fits in an unsigned 64-bit integer. - * - * Equivalent to is(). - */ - inline bool is_uint64() const noexcept; - /** - * Whether this element is a json number that fits in a double. - * - * Equivalent to is(). - */ - inline bool is_double() const noexcept; - - /** - * Whether this element is a json number. - * - * Both integers and floating points will return true. - */ - inline bool is_number() const noexcept; - - /** - * Whether this element is a json `true` or `false`. - * - * Equivalent to is(). - */ - inline bool is_bool() const noexcept; - /** - * Whether this element is a json `null`. - */ - inline bool is_null() const noexcept; - - /** - * Whether this element is a big integer (number exceeding 64-bit range). - */ - inline bool is_bigint() const noexcept; - - /** - * Tell whether the value can be cast to provided type (T). - * - * Supported types: - * - Boolean: bool - * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char * - * - Array: dom::array - * - Object: dom::object - * - * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object - */ - template - simdjson_inline bool is() const noexcept; - - /** - * Get the value as the provided type (T). - * - * Supported types: - * - Boolean: bool - * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char * - * - Array: dom::array - * - Object: dom::object - * - * You may use get_double(), get_bool(), get_uint64(), get_int64(), - * get_object(), get_array() or get_string() instead. - * - * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object - * - * @returns The value cast to the given type, or: - * INCORRECT_TYPE if the value cannot be cast to the given type. - */ - - template - inline simdjson_result get() const noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should - // immediately fail. - static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " - "The supported types are Boolean (bool), numbers (double, uint64_t, int64_t), " - "strings (std::string_view, const char *), arrays (dom::array) and objects (dom::object). " - "We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - "get_object(), get_array() or get_string() instead of the get template."); - } - - /** - * Get the value as the provided type (T). - * - * Supported types: - * - Boolean: bool - * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char * - * - Array: dom::array - * - Object: dom::object - * - * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object - * - * @param value The variable to set to the value. May not be set if there is an error. - * - * @returns The error that occurred, or SUCCESS if there was no error. - */ - template - simdjson_warn_unused simdjson_inline error_code get(T &value) const noexcept; - - /** - * Get the value as the provided type (T), setting error if it's not the given type. - * - * Supported types: - * - Boolean: bool - * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char * - * - Array: dom::array - * - Object: dom::object - * - * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object - * - * @param value The variable to set to the given type. value is undefined if there is an error. - * @param error The variable to store the error. error is set to error_code::SUCCEED if there is an error. - */ - template - inline void tie(T &value, error_code &error) && noexcept; - -#if SIMDJSON_EXCEPTIONS - /** - * Read this element as a boolean. - * - * @return The boolean value - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not a boolean. - */ - inline operator bool() const noexcept(false); - - /** - * Read this element as a null-terminated UTF-8 string. - * - * Be mindful that JSON allows strings to contain null characters. - * - * Does *not* convert other types to a string; requires that the JSON type of the element was - * an actual string. - * - * @return The string value. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not a string. - */ - inline explicit operator const char*() const noexcept(false); - - /** - * Read this element as a null-terminated UTF-8 string. - * - * Does *not* convert other types to a string; requires that the JSON type of the element was - * an actual string. - * - * @return The string value. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not a string. - */ - inline operator std::string_view() const noexcept(false); - - /** - * Read this element as an unsigned integer. - * - * @return The integer value. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an integer - * @exception simdjson_error(NUMBER_OUT_OF_RANGE) if the integer does not fit in 64 bits or is negative - */ - inline operator uint64_t() const noexcept(false); - /** - * Read this element as an signed integer. - * - * @return The integer value. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an integer - * @exception simdjson_error(NUMBER_OUT_OF_RANGE) if the integer does not fit in 64 bits - */ - inline operator int64_t() const noexcept(false); - /** - * Read this element as an double. - * - * @return The double value. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not a number - */ - inline operator double() const noexcept(false); - /** - * Read this element as a JSON array. - * - * @return The JSON array. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an array - */ - inline operator array() const noexcept(false); - /** - * Read this element as a JSON object (key/value pairs). - * - * @return The JSON object. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an object - */ - inline operator object() const noexcept(false); - - /** - * Iterate over each element in this array. - * - * @return The beginning of the iteration. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an array - */ - inline dom::array::iterator begin() const noexcept(false); - - /** - * Iterate over each element in this array. - * - * @return The end of the iteration. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an array - */ - inline dom::array::iterator end() const noexcept(false); -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the value associated with the given key. - * - * The key will be matched against **unescaped** JSON: - * - * dom::parser parser; - * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 - * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD - * - * @return The value associated with this field, or: - * - NO_SUCH_FIELD if the field does not exist in the object - * - INCORRECT_TYPE if this is not an object - */ - inline simdjson_result operator[](std::string_view key) const noexcept; - - /** - * Get the value associated with the given key. - * - * The key will be matched against **unescaped** JSON: - * - * dom::parser parser; - * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 - * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD - * - * @return The value associated with this field, or: - * - NO_SUCH_FIELD if the field does not exist in the object - * - INCORRECT_TYPE if this is not an object - */ - inline simdjson_result operator[](const char *key) const noexcept; - simdjson_result operator[](int) const noexcept = delete; - - - /** - * Get the value associated with the given JSON pointer. We use the RFC 6901 - * https://tools.ietf.org/html/rfc6901 standard. - * - * dom::parser parser; - * element doc = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded); - * doc.at_pointer("/foo/a/1") == 20 - * doc.at_pointer("/foo")["a"].at(1) == 20 - * doc.at_pointer("")["foo"]["a"].at(1) == 20 - * - * It is allowed for a key to be the empty string: - * - * dom::parser parser; - * object obj = parser.parse(R"({ "": { "a": [ 10, 20, 30 ] }})"_padded); - * obj.at_pointer("//a/1") == 20 - * - * @return The value associated with the given JSON pointer, or: - * - NO_SUCH_FIELD if a field does not exist in an object - * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length - * - INCORRECT_TYPE if a non-integer is used to access an array - * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed - */ - inline simdjson_result at_pointer(const std::string_view json_pointer) const noexcept; - - inline simdjson_result> at_path_with_wildcard(const std::string_view json_path) const noexcept; - - /** - * Get the value associated with the given JSONPath expression. We only support - * JSONPath queries that trivially convertible to JSON Pointer queries: key - * names and array indices. - * - * https://www.rfc-editor.org/rfc/rfc9535 (RFC 9535) - * - * @return The value associated with the given JSONPath expression, or: - * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails - * - NO_SUCH_FIELD if a field does not exist in an object - * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length - * - INCORRECT_TYPE if a non-integer is used to access an array - */ - inline simdjson_result at_path(std::string_view json_path) const noexcept; - -#ifndef SIMDJSON_DISABLE_DEPRECATED_API - /** - * - * Version 0.4 of simdjson used an incorrect interpretation of the JSON Pointer standard - * and allowed the following : - * - * dom::parser parser; - * element doc = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded); - * doc.at("foo/a/1") == 20 - * - * Though it is intuitive, it is not compliant with RFC 6901 - * https://tools.ietf.org/html/rfc6901 - * - * For standard compliance, use the at_pointer function instead. - * - * @return The value associated with the given JSON pointer, or: - * - NO_SUCH_FIELD if a field does not exist in an object - * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length - * - INCORRECT_TYPE if a non-integer is used to access an array - * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed - */ - [[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] - inline simdjson_result at(const std::string_view json_pointer) const noexcept; -#endif // SIMDJSON_DISABLE_DEPRECATED_API - - /** - * Get the value at the given index. - * - * @return The value at the given index, or: - * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length - */ - inline simdjson_result at(size_t index) const noexcept; - - /** - * Get the value associated with the given key. - * - * The key will be matched against **unescaped** JSON: - * - * dom::parser parser; - * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 - * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD - * - * @return The value associated with this field, or: - * - NO_SUCH_FIELD if the field does not exist in the object - */ - inline simdjson_result at_key(std::string_view key) const noexcept; - - /** - * Get the value associated with the given key in a case-insensitive manner. - * - * Note: The key will be matched against **unescaped** JSON. - * - * @return The value associated with this field, or: - * - NO_SUCH_FIELD if the field does not exist in the object - */ - inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; - - /** - * operator< defines a total order for element allowing to use them in - * ordered C++ STL containers - * - * @return TRUE if the key appears before the other one in the tape - */ - inline bool operator<(const element &other) const noexcept; - - /** - * operator== allows to verify if two element values reference the - * same JSON item - * - * @return TRUE if the two values references the same JSON element - */ - inline bool operator==(const element &other) const noexcept; - - /** @private for debugging. Prints out the root element. */ - inline bool dump_raw_tape(std::ostream &out) const noexcept; - -private: - simdjson_inline element(const internal::tape_ref &tape) noexcept; - internal::tape_ref tape{}; - friend class document; - friend class object; - friend class array; - friend struct simdjson_result; - template - friend class simdjson::internal::string_builder; - -}; - -} // namespace dom - -/** The result of a JSON navigation that may fail. */ -template<> -struct simdjson_result : public internal::simdjson_result_base { -public: - simdjson_inline simdjson_result() noexcept; ///< @private - simdjson_inline simdjson_result(dom::element &&value) noexcept; ///< @private - simdjson_inline simdjson_result(error_code error) noexcept; ///< @private - - simdjson_inline simdjson_result type() const noexcept; - template - simdjson_inline bool is() const noexcept; - template - simdjson_inline simdjson_result get() const noexcept; - template - simdjson_warn_unused simdjson_inline error_code get(T &value) const noexcept; - - simdjson_inline simdjson_result get_array() const noexcept; - simdjson_inline simdjson_result get_object() const noexcept; - simdjson_inline simdjson_result get_c_str() const noexcept; - simdjson_inline simdjson_result get_string_length() const noexcept; - simdjson_inline simdjson_result get_string() const noexcept; - simdjson_inline simdjson_result get_int64() const noexcept; - simdjson_inline simdjson_result get_uint64() const noexcept; - simdjson_inline simdjson_result get_double() const noexcept; - simdjson_inline simdjson_result get_bool() const noexcept; - simdjson_inline simdjson_result get_bigint() const noexcept; - - simdjson_inline bool is_array() const noexcept; - simdjson_inline bool is_object() const noexcept; - simdjson_inline bool is_string() const noexcept; - simdjson_inline bool is_int64() const noexcept; - simdjson_inline bool is_uint64() const noexcept; - simdjson_inline bool is_double() const noexcept; - simdjson_inline bool is_number() const noexcept; - simdjson_inline bool is_bool() const noexcept; - simdjson_inline bool is_null() const noexcept; - simdjson_inline bool is_bigint() const noexcept; - - simdjson_inline simdjson_result operator[](std::string_view key) const noexcept; - simdjson_inline simdjson_result operator[](const char *key) const noexcept; - simdjson_result operator[](int) const noexcept = delete; - simdjson_inline simdjson_result at_pointer(const std::string_view json_pointer) const noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(const std::string_view json_path) const noexcept; - simdjson_inline simdjson_result at_path(const std::string_view json_path) const noexcept; - [[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] - simdjson_inline simdjson_result at(const std::string_view json_pointer) const noexcept; - simdjson_inline simdjson_result at(size_t index) const noexcept; - simdjson_inline simdjson_result at_key(std::string_view key) const noexcept; - simdjson_inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; - -#if SIMDJSON_EXCEPTIONS - simdjson_inline operator bool() const noexcept(false); - simdjson_inline explicit operator const char*() const noexcept(false); - simdjson_inline operator std::string_view() const noexcept(false); - simdjson_inline operator uint64_t() const noexcept(false); - simdjson_inline operator int64_t() const noexcept(false); - simdjson_inline operator double() const noexcept(false); - simdjson_inline operator dom::array() const noexcept(false); - simdjson_inline operator dom::object() const noexcept(false); - - simdjson_inline dom::array::iterator begin() const noexcept(false); - simdjson_inline dom::array::iterator end() const noexcept(false); -#endif // SIMDJSON_EXCEPTIONS -}; - -} // namespace simdjson - -#endif // SIMDJSON_DOM_DOCUMENT_H -/* end file simdjson/dom/element.h */ -/* including simdjson/dom/object.h: #include "simdjson/dom/object.h" */ -/* begin file simdjson/dom/object.h */ -#ifndef SIMDJSON_DOM_OBJECT_H -#define SIMDJSON_DOM_OBJECT_H - -#include - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/element.h" */ -/* skipped duplicate #include "simdjson/internal/tape_ref.h" */ - -namespace simdjson { -namespace dom { - -/** - * JSON object. - */ -class object { -public: - /** Create a new, invalid object */ - simdjson_inline object() noexcept; - - class iterator { - public: - using value_type = const key_value_pair; - using difference_type = std::ptrdiff_t; - using pointer = void; - using reference = value_type; - using iterator_category = std::forward_iterator_tag; - - /** - * Get the actual key/value pair - */ - inline reference operator*() const noexcept; - /** - * Get the next key/value pair. - * - * Part of the std::iterator interface. - * - */ - inline iterator& operator++() noexcept; - /** - * Get the next key/value pair. - * - * Part of the std::iterator interface. - * - */ - inline iterator operator++(int) noexcept; - /** - * Check if these values come from the same place in the JSON. - * - * Part of the std::iterator interface. - */ - inline bool operator!=(const iterator& other) const noexcept; - inline bool operator==(const iterator& other) const noexcept; - - inline bool operator<(const iterator& other) const noexcept; - inline bool operator<=(const iterator& other) const noexcept; - inline bool operator>=(const iterator& other) const noexcept; - inline bool operator>(const iterator& other) const noexcept; - /** - * Get the key of this key/value pair. - */ - inline std::string_view key() const noexcept; - /** - * Get the length (in bytes) of the key in this key/value pair. - * You should expect this function to be faster than key().size(). - */ - inline uint32_t key_length() const noexcept; - /** - * Returns true if the key in this key/value pair is equal - * to the provided string_view. - */ - inline bool key_equals(std::string_view o) const noexcept; - /** - * Returns true if the key in this key/value pair is equal - * to the provided string_view in a case-insensitive manner. - * Case comparisons may only be handled correctly for ASCII strings. - */ - inline bool key_equals_case_insensitive(std::string_view o) const noexcept; - /** - * Get the key of this key/value pair. - */ - inline const char *key_c_str() const noexcept; - /** - * Get the value of this key/value pair. - */ - inline element value() const noexcept; - - iterator() noexcept = default; - iterator(const iterator&) noexcept = default; - iterator& operator=(const iterator&) noexcept = default; - private: - simdjson_inline iterator(const internal::tape_ref &tape) noexcept; - - internal::tape_ref tape{}; - - friend class object; - }; - - /** - * Return the first key/value pair. - * - * Part of the std::iterable interface. - */ - inline iterator begin() const noexcept; - /** - * One past the last key/value pair. - * - * Part of the std::iterable interface. - */ - inline iterator end() const noexcept; - /** - * Get the size of the object (number of keys). - * It is a saturated value with a maximum of 0xFFFFFF: if the value - * is 0xFFFFFF then the size is 0xFFFFFF or greater. - */ - inline size_t size() const noexcept; - /** - * Get the value associated with the given key. - * - * The key will be matched against **unescaped** JSON: - * - * dom::parser parser; - * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 - * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD - * - * This function has linear-time complexity: the keys are checked one by one. - * - * @return The value associated with this field, or: - * - NO_SUCH_FIELD if the field does not exist in the object - * - INCORRECT_TYPE if this is not an object - */ - inline simdjson_result operator[](std::string_view key) const noexcept; - - /** - * Get the value associated with the given key. - * - * The key will be matched against **unescaped** JSON: - * - * dom::parser parser; - * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 - * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD - * - * This function has linear-time complexity: the keys are checked one by one. - * - * @return The value associated with this field, or: - * - NO_SUCH_FIELD if the field does not exist in the object - * - INCORRECT_TYPE if this is not an object - */ - inline simdjson_result operator[](const char *key) const noexcept; - simdjson_result operator[](int) const noexcept = delete; - - /** - * Get the value associated with the given JSON pointer. We use the RFC 6901 - * https://tools.ietf.org/html/rfc6901 standard, interpreting the current node - * as the root of its own JSON document. - * - * dom::parser parser; - * object obj = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded); - * obj.at_pointer("/foo/a/1") == 20 - * obj.at_pointer("/foo")["a"].at(1) == 20 - * - * It is allowed for a key to be the empty string: - * - * dom::parser parser; - * object obj = parser.parse(R"({ "": { "a": [ 10, 20, 30 ] }})"_padded); - * obj.at_pointer("//a/1") == 20 - * obj.at_pointer("/")["a"].at(1) == 20 - * - * @return The value associated with the given JSON pointer, or: - * - NO_SUCH_FIELD if a field does not exist in an object - * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length - * - INCORRECT_TYPE if a non-integer is used to access an array - * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed - */ - inline simdjson_result at_pointer(std::string_view json_pointer) const noexcept; - - /** - * Recursive function which processes the JSON path of each child element - */ - inline void process_json_path_of_child_elements(std::vector::iterator& current, std::vector::iterator& end, const std::string_view& path_suffix, std::vector& accumulator) const noexcept; - - /** - * Adds support for JSONPath expression with wildcards '*' - */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) const noexcept; - - /** - * Get the value associated with the given JSONPath expression. We only support - * JSONPath queries that trivially convertible to JSON Pointer queries: key - * names and array indices. - * - * https://www.rfc-editor.org/rfc/rfc9535 (RFC 9535) - * - * @return The value associated with the given JSONPath expression, or: - * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails - * - NO_SUCH_FIELD if a field does not exist in an object - * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length - * - INCORRECT_TYPE if a non-integer is used to access an array - */ - inline simdjson_result at_path(std::string_view json_path) const noexcept; - - /** - * Get the value associated with the given key. - * - * The key will be matched against **unescaped** JSON: - * - * dom::parser parser; - * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 - * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD - * - * This function has linear-time complexity: the keys are checked one by one. - * - * @return The value associated with this field, or: - * - NO_SUCH_FIELD if the field does not exist in the object - */ - inline simdjson_result at_key(std::string_view key) const noexcept; - - /** - * Gets the values associated with keys of an object - * This function has linear-time complexity: the keys are checked one by one. - * - * @return the values associated with each key of an object - */ - inline std::vector& get_values(std::vector& out) const noexcept; - - /** - * Get the value associated with the given key in a case-insensitive manner. - * It is only guaranteed to work over ASCII inputs. - * - * Note: The key will be matched against **unescaped** JSON. - * - * This function has linear-time complexity: the keys are checked one by one. - * - * @return The value associated with this field, or: - * - NO_SUCH_FIELD if the field does not exist in the object - */ - inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; - - /** - * Implicitly convert object to element - */ - inline operator element() const noexcept; - -private: - simdjson_inline object(const internal::tape_ref &tape) noexcept; - - internal::tape_ref tape{}; - - friend class element; - friend struct simdjson_result; - template - friend class simdjson::internal::string_builder; -}; - -/** - * Key/value pair in an object. - */ -class key_value_pair { -public: - /** key in the key-value pair **/ - std::string_view key; - /** value in the key-value pair **/ - element value; - -private: - simdjson_inline key_value_pair(std::string_view _key, element _value) noexcept; - friend class object; -}; - -} // namespace dom - -/** The result of a JSON conversion that may fail. */ -template<> -struct simdjson_result : public internal::simdjson_result_base { -public: - simdjson_inline simdjson_result() noexcept; ///< @private - simdjson_inline simdjson_result(dom::object value) noexcept; ///< @private - simdjson_inline simdjson_result(error_code error) noexcept; ///< @private - - inline simdjson_result operator[](std::string_view key) const noexcept; - inline simdjson_result operator[](const char *key) const noexcept; - simdjson_result operator[](int) const noexcept = delete; - inline simdjson_result at_pointer(std::string_view json_pointer) const noexcept; - inline void process_json_path_of_child_elements(std::vector::iterator& current, std::vector::iterator& end, const std::string_view& path_suffix, std::vector& accumulator) const noexcept; - inline simdjson_result> at_path_with_wildcard(std::string_view json_path_new) const noexcept; - inline simdjson_result at_path(std::string_view json_path) const noexcept; - inline simdjson_result at_key(std::string_view key) const noexcept; - inline std::vector& get_values(std::vector& out) const noexcept; - inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; - -#if SIMDJSON_EXCEPTIONS - inline dom::object::iterator begin() const noexcept(false); - inline dom::object::iterator end() const noexcept(false); - inline size_t size() const noexcept(false); -#endif // SIMDJSON_EXCEPTIONS -}; - -} // namespace simdjson - -#if SIMDJSON_SUPPORTS_RANGES -namespace std { -namespace ranges { -template<> -inline constexpr bool enable_view = true; -#if SIMDJSON_EXCEPTIONS -template<> -inline constexpr bool enable_view> = true; -#endif // SIMDJSON_EXCEPTIONS -} // namespace ranges -} // namespace std -#endif // SIMDJSON_SUPPORTS_RANGES - -#endif // SIMDJSON_DOM_OBJECT_H -/* end file simdjson/dom/object.h */ -/* skipped duplicate #include "simdjson/dom/parser.h" */ -/* including simdjson/dom/serialization.h: #include "simdjson/dom/serialization.h" */ -/* begin file simdjson/dom/serialization.h */ -#ifndef SIMDJSON_SERIALIZATION_H -#define SIMDJSON_SERIALIZATION_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/element.h" */ -/* skipped duplicate #include "simdjson/dom/object.h" */ - -namespace simdjson { - -/** - * The string_builder template and mini_formatter class - * are not part of our public API and are subject to change - * at any time! - */ -namespace internal { - -template class base_formatter { -public: - /** Add a comma **/ - simdjson_inline void comma(); - /** Start an array, prints [ **/ - simdjson_inline void start_array(); - /** End an array, prints ] **/ - simdjson_inline void end_array(); - /** Start an array, prints { **/ - simdjson_inline void start_object(); - /** Start an array, prints } **/ - simdjson_inline void end_object(); - /** Prints a true **/ - simdjson_inline void true_atom(); - /** Prints a false **/ - simdjson_inline void false_atom(); - /** Prints a null **/ - simdjson_inline void null_atom(); - /** Prints a number **/ - simdjson_inline void number(int64_t x); - /** Prints a number **/ - simdjson_inline void number(uint64_t x); - /** Prints a number **/ - simdjson_inline void number(double x); - /** Prints a key (string + colon) **/ - simdjson_inline void key(std::string_view unescaped); - /** Prints a string. The string is escaped as needed. **/ - simdjson_inline void string(std::string_view unescaped); - /** Clears out the content. **/ - simdjson_inline void clear(); - /** - * Get access to the buffer, it is owned by the instance, but - * the user can make a copy. - **/ - simdjson_inline std::string_view str() const; - - /** Prints one character **/ - simdjson_inline void one_char(char c); - - /** Prints characters in [begin, end) verbatim. **/ - simdjson_inline void chars(const char *begin, const char *end); - - simdjson_inline void call_print_newline() { - static_cast(this)->print_newline(); - } - - simdjson_inline void call_print_indents(size_t depth) { - static_cast(this)->print_indents(depth); - } - - simdjson_inline void call_print_space() { - static_cast(this)->print_space(); - } - -protected: - // implementation details (subject to change) - /** Backing buffer **/ - struct vector_with_small_buffer { - vector_with_small_buffer() = default; - ~vector_with_small_buffer() { free_buffer(); } - - vector_with_small_buffer(const vector_with_small_buffer &) = delete; - vector_with_small_buffer & - operator=(const vector_with_small_buffer &) = delete; - - void clear() { - size = 0; - capacity = StaticCapacity; - free_buffer(); - buffer = array; - } - - simdjson_inline void push_back(char c) { - if (capacity < size + 1) - grow(capacity * 2); - buffer[size++] = c; - } - - simdjson_inline void append(const char *begin, const char *end) { - const size_t new_size = size + (end - begin); - if (capacity < new_size) - // std::max(new_size, capacity * 2); is broken in tests on Windows - grow(new_size < capacity * 2 ? capacity * 2 : new_size); - std::copy(begin, end, buffer + size); - size = new_size; - } - - std::string_view str() const { return std::string_view(buffer, size); } - - private: - void free_buffer() { - if (buffer != array) - delete[] buffer; - } - void grow(size_t new_capacity) { - auto new_buffer = new char[new_capacity]; - std::copy(buffer, buffer + size, new_buffer); - free_buffer(); - buffer = new_buffer; - capacity = new_capacity; - } - - static const size_t StaticCapacity = 64; - char array[StaticCapacity]; - char *buffer = array; - size_t size = 0; - size_t capacity = StaticCapacity; - } buffer{}; -}; - -/** - * @private This is the class that we expect to use with the string_builder - * template. It tries to produce a compact version of the JSON element - * as quickly as possible. - */ -class mini_formatter : public base_formatter { -public: - simdjson_inline void print_newline(); - - simdjson_inline void print_indents(size_t depth); - - simdjson_inline void print_space(); -}; - -class pretty_formatter : public base_formatter { -public: - simdjson_inline void print_newline(); - - simdjson_inline void print_indents(size_t depth); - - simdjson_inline void print_space(); - -protected: - int indent_step = 4; -}; - -/** - * @private The string_builder template allows us to construct - * a string from a document element. It is parametrized - * by a "formatter" which handles the details. Thus - * the string_builder template could support both minification - * and prettification, and various other tradeoffs. - * - * This is not to be confused with the simdjson::builder::string_builder - * which is a different class. - */ -template class string_builder { -public: - /** Construct an initially empty builder, would print the empty string **/ - string_builder() = default; - /** Append an element to the builder (to be printed) **/ - inline void append(simdjson::dom::element value); - /** Append an array to the builder (to be printed) **/ - inline void append(simdjson::dom::array value); - /** Append an object to the builder (to be printed) **/ - inline void append(simdjson::dom::object value); - /** Reset the builder (so that it would print the empty string) **/ - simdjson_inline void clear(); - /** - * Get access to the string. The string_view is owned by the builder - * and it is invalid to use it after the string_builder has been - * destroyed. - * However you can make a copy of the string_view on memory that you - * own. - */ - simdjson_inline std::string_view str() const; - /** Append a key_value_pair to the builder (to be printed) **/ - simdjson_inline void append(simdjson::dom::key_value_pair value); - -private: - formatter format{}; -}; - -} // namespace internal - -namespace dom { - -/** - * Print JSON to an output stream. - * - * @param out The output stream. - * @param value The element. - * @throw if there is an error with the underlying output stream. simdjson - * itself will not throw. - */ -inline std::ostream &operator<<(std::ostream &out, - simdjson::dom::element value); -#if SIMDJSON_EXCEPTIONS -inline std::ostream & -operator<<(std::ostream &out, - simdjson::simdjson_result x); -#endif -/** - * Print JSON to an output stream. - * - * @param out The output stream. - * @param value The array. - * @throw if there is an error with the underlying output stream. simdjson - * itself will not throw. - */ -inline std::ostream &operator<<(std::ostream &out, simdjson::dom::array value); -#if SIMDJSON_EXCEPTIONS -inline std::ostream & -operator<<(std::ostream &out, - simdjson::simdjson_result x); -#endif -/** - * Print JSON to an output stream. - * - * @param out The output stream. - * @param value The object. - * @throw if there is an error with the underlying output stream. simdjson - * itself will not throw. - */ -inline std::ostream &operator<<(std::ostream &out, simdjson::dom::object value); -#if SIMDJSON_EXCEPTIONS -inline std::ostream & -operator<<(std::ostream &out, - simdjson::simdjson_result x); -#endif -} // namespace dom - -/** - * Converts JSON to a string. - * - * dom::parser parser; - * element doc = parser.parse(" [ 1 , 2 , 3 ] "_padded); - * cout << to_string(doc) << endl; // prints [1,2,3] - * - */ -template std::string to_string(T x) { - // in C++, to_string is standard: - // http://www.cplusplus.com/reference/string/to_string/ Currently minify and - // to_string are identical but in the future, they may differ. - simdjson::internal::string_builder<> sb; - sb.append(x); - std::string_view answer = sb.str(); - return std::string(answer.data(), answer.size()); -} -#if SIMDJSON_EXCEPTIONS -template std::string to_string(simdjson_result x) { - if (x.error()) { - throw simdjson_error(x.error()); - } - return to_string(x.value()); -} -#endif - -/** - * Minifies a JSON element or document, printing the smallest possible valid - * JSON. - * - * dom::parser parser; - * element doc = parser.parse(" [ 1 , 2 , 3 ] "_padded); - * cout << minify(doc) << endl; // prints [1,2,3] - * - */ -template std::string minify(T x) { return to_string(x); } - -#if SIMDJSON_EXCEPTIONS -template std::string minify(simdjson_result x) { - if (x.error()) { - throw simdjson_error(x.error()); - } - return to_string(x.value()); -} -#endif - -/** - * Prettifies a JSON element or document, printing the valid JSON with - * indentation. - * - * dom::parser parser; - * element doc = parser.parse(" [ 1 , 2 , 3 ] "_padded); - * - * // Prints: - * // { - * // [ - * // 1, - * // 2, - * // 3 - * // ] - * // } - * cout << prettify(doc) << endl; - * - */ -template std::string prettify(T x) { - simdjson::internal::string_builder sb; - sb.append(x); - std::string_view answer = sb.str(); - return std::string(answer.data(), answer.size()); -} - -#if SIMDJSON_EXCEPTIONS -template std::string prettify(simdjson_result x) { - if (x.error()) { - throw simdjson_error(x.error()); - } - return to_string(x.value()); -} -#endif - -} // namespace simdjson - -#endif -/* end file simdjson/dom/serialization.h */ -/* including simdjson/dom/fractured_json.h: #include "simdjson/dom/fractured_json.h" */ -/* begin file simdjson/dom/fractured_json.h */ -#ifndef SIMDJSON_DOM_FRACTURED_JSON_H -#define SIMDJSON_DOM_FRACTURED_JSON_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/element.h" */ - -namespace simdjson { - -/** - * Configuration options for FracturedJson formatting. - * - * FracturedJson intelligently chooses between different layout strategies - * (inline, compact multiline, table, expanded) based on content complexity, - * length, and structure similarity. - */ -struct fractured_json_options { - /** - * Maximum total characters per line (default: 120). - * Content exceeding this will be expanded to multiple lines. - */ - size_t max_total_line_length = 120; - - /** - * Maximum length for inlined elements (default: 80). - * Simple arrays/objects shorter than this may be rendered inline. - */ - size_t max_inline_length = 80; - - /** - * Maximum nesting depth for inline rendering (default: 2). - * Elements with complexity exceeding this will be expanded. - * Complexity 0 = scalar, 1 = flat array/object, 2 = one level of nesting. - */ - size_t max_inline_complexity = 2; - - /** - * Maximum complexity for compact array formatting (default: 1). - * Arrays with elements of this complexity or less may have multiple - * items per line. - */ - size_t max_compact_array_complexity = 1; - - /** - * Number of spaces per indentation level (default: 4). - */ - size_t indent_spaces = 4; - - /** - * Enable tabular formatting for arrays of similar objects (default: true). - * When enabled, arrays of objects with identical keys are formatted - * as aligned tables. - */ - bool enable_table_format = true; - - /** - * Minimum number of rows to trigger table mode (default: 3). - */ - size_t min_table_rows = 3; - - /** - * Similarity threshold for table detection (default: 0.8). - * Objects must share at least this fraction of keys to be formatted - * as a table. - */ - double table_similarity_threshold = 0.8; - - /** - * Enable compact multiline arrays (default: true). - * When enabled, arrays of simple elements may have multiple items - * per line. - */ - bool enable_compact_multiline = true; - - /** - * Maximum array items per line in compact mode (default: 10). - */ - size_t max_items_per_line = 10; - - /** - * Add space inside brackets for simple containers (default: true). - * When true: { "key": "value" } - * When false: {"key": "value"} - */ - bool simple_bracket_padding = true; - - /** - * Add space after colons (default: true). - * When true: "key": "value" - * When false: "key":"value" - */ - bool colon_padding = true; - - /** - * Add space after commas in inline content (default: true). - * When true: [1, 2, 3] - * When false: [1,2,3] - */ - bool comma_padding = true; -}; - -/** - * Format JSON using FracturedJson formatting with default options. - * - * FracturedJson produces human-readable yet compact output by intelligently - * choosing between inline, compact multiline, table, and expanded layouts. - * - * dom::parser parser; - * element doc = parser.parse(json_string); - * cout << fractured_json(doc) << endl; - */ -template -std::string fractured_json(T x); - -/** - * Format JSON using FracturedJson formatting with custom options. - * - * dom::parser parser; - * element doc = parser.parse(json_string); - * fractured_json_options opts; - * opts.max_total_line_length = 80; - * cout << fractured_json(doc, opts) << endl; - */ -template -std::string fractured_json(T x, const fractured_json_options& options); - -#if SIMDJSON_EXCEPTIONS -template -std::string fractured_json(simdjson_result x); - -template -std::string fractured_json(simdjson_result x, const fractured_json_options& options); -#endif - -/** - * Format a JSON string using FracturedJson formatting. - * - * This is useful for formatting output from the builder/static reflection API - * or any valid JSON string. - * - * // With static reflection - * MyStruct data = {...}; - * auto minified = simdjson::to_json_string(data); - * auto formatted = simdjson::fractured_json_string(minified.value()); - * - * // Or with any JSON string - * std::string json = R"({"key":"value"})"; - * auto formatted = simdjson::fractured_json_string(json); - */ -inline std::string fractured_json_string(std::string_view json_str); - -/** - * Format a JSON string using FracturedJson formatting with custom options. - */ -inline std::string fractured_json_string(std::string_view json_str, - const fractured_json_options& options); - -} // namespace simdjson - -#endif // SIMDJSON_DOM_FRACTURED_JSON_H -/* end file simdjson/dom/fractured_json.h */ - -// Inline functions -/* including simdjson/dom/array-inl.h: #include "simdjson/dom/array-inl.h" */ -/* begin file simdjson/dom/array-inl.h */ -#ifndef SIMDJSON_ARRAY_INL_H -#define SIMDJSON_ARRAY_INL_H - -#include - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/array.h" */ -/* skipped duplicate #include "simdjson/dom/element.h" */ -/* skipped duplicate #include "simdjson/error-inl.h" */ -/* including simdjson/jsonpathutil.h: #include "simdjson/jsonpathutil.h" */ -/* begin file simdjson/jsonpathutil.h */ -#ifndef SIMDJSON_JSONPATHUTIL_H -#define SIMDJSON_JSONPATHUTIL_H - -#include -/* skipped duplicate #include "simdjson/common_defs.h" */ - -#include - -namespace simdjson { -/** - * Converts JSONPath to JSON Pointer. - * @param json_path The JSONPath string to be converted. - * @return A string containing the equivalent JSON Pointer. - */ -inline std::string json_path_to_pointer_conversion(std::string_view json_path) { - size_t i = 0; - // if JSONPath starts with $, skip it - // json_path.starts_with('$') requires C++20. - if (!json_path.empty() && json_path.front() == '$') { - i = 1; - } - if (i >= json_path.size() || (json_path[i] != '.' && - json_path[i] != '[')) { - return "-1"; // This is just a sentinel value, the caller should check for this and return an error. - } - - std::string result; - // Reserve space to reduce allocations, adjusting for potential increases due - // to escaping. - result.reserve(json_path.size() * 2); - - while (i < json_path.length()) { - if (json_path[i] == '.') { - result += '/'; - } else if (json_path[i] == '[') { - result += '/'; - ++i; // Move past the '[' - while (i < json_path.length() && json_path[i] != ']') { - if (json_path[i] == '~') { - result += "~0"; - } else if (json_path[i] == '/') { - result += "~1"; - } else { - result += json_path[i]; - } - ++i; - } - if (i == json_path.length() || json_path[i] != ']') { - return "-1"; // Using sentinel value that will be handled as an error by the caller. - } - } else { - if (json_path[i] == '~') { - result += "~0"; - } else if (json_path[i] == '/') { - result += "~1"; - } else { - result += json_path[i]; - } - } - ++i; - } - - return result; -} - -inline std::pair get_next_key_and_json_path(std::string_view& json_path) { - std::string_view key; - - if (json_path.empty()) { - return {key, json_path}; - } - size_t i = 0; - - // if JSONPath starts with $, skip it - if (json_path.front() == '$') { - i = 1; - } - - - if (i < json_path.length() && json_path[i] == '.') { - i += 1; - size_t key_start = i; - - while (i < json_path.length() && json_path[i] != '[' && json_path[i] != '.') { - ++i; - } - - key = json_path.substr(key_start, i - key_start); - } else if ((i+1 < json_path.size()) && json_path[i] == '[' && (json_path[i+1] == '\'' || json_path[i+1] == '"')) { - i += 2; - size_t key_start = i; - while (i < json_path.length() && json_path[i] != '\'' && json_path[i] != '"') { - ++i; - } - - key = json_path.substr(key_start, i - key_start); - - i += 2; - } else if ((i+2 < json_path.size()) && json_path[i] == '[' && json_path[i+1] == '*' && json_path[i+2] == ']') { // i.e [*].additional_keys or [*]["additional_keys"] - key = "*"; - i += 3; - } - - - return std::make_pair(key, json_path.substr(i)); -} - -} // namespace simdjson -#endif // SIMDJSON_JSONPATHUTIL_H -/* end file simdjson/jsonpathutil.h */ -/* including simdjson/internal/tape_ref-inl.h: #include "simdjson/internal/tape_ref-inl.h" */ -/* begin file simdjson/internal/tape_ref-inl.h */ -#ifndef SIMDJSON_TAPE_REF_INL_H -#define SIMDJSON_TAPE_REF_INL_H - -/* skipped duplicate #include "simdjson/dom/document.h" */ -/* skipped duplicate #include "simdjson/internal/tape_ref.h" */ -/* including simdjson/internal/tape_type.h: #include "simdjson/internal/tape_type.h" */ -/* begin file simdjson/internal/tape_type.h */ -#ifndef SIMDJSON_INTERNAL_TAPE_TYPE_H -#define SIMDJSON_INTERNAL_TAPE_TYPE_H - -namespace simdjson { -namespace internal { - -/** - * The possible types in the tape. - */ -enum class tape_type { - ROOT = 'r', - START_ARRAY = '[', - START_OBJECT = '{', - END_ARRAY = ']', - END_OBJECT = '}', - STRING = '"', - INT64 = 'l', - UINT64 = 'u', - DOUBLE = 'd', - TRUE_VALUE = 't', - FALSE_VALUE = 'f', - NULL_VALUE = 'n', - BIGINT = 'Z' // Big integer stored as string in string buffer -}; // enum class tape_type - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_TAPE_TYPE_H -/* end file simdjson/internal/tape_type.h */ - -#include - -namespace simdjson { -namespace internal { - -constexpr const uint64_t JSON_VALUE_MASK = 0x00FFFFFFFFFFFFFF; -constexpr const uint32_t JSON_COUNT_MASK = 0xFFFFFF; - -// -// tape_ref inline implementation -// -simdjson_inline tape_ref::tape_ref() noexcept : doc{nullptr}, json_index{0} {} -simdjson_inline tape_ref::tape_ref(const dom::document *_doc, size_t _json_index) noexcept : doc{_doc}, json_index{_json_index} {} - - -simdjson_inline bool tape_ref::is_document_root() const noexcept { - return json_index == 1; // should we ever change the structure of the tape, this should get updated. -} -simdjson_inline bool tape_ref::usable() const noexcept { - return doc != nullptr; // when the document pointer is null, this tape_ref is uninitialized (should not be accessed). -} -// Some value types have a specific on-tape word value. It can be faster -// to check the type by doing a word-to-word comparison instead of extracting the -// most significant 8 bits. - -simdjson_inline bool tape_ref::is_double() const noexcept { - constexpr uint64_t tape_double = uint64_t(tape_type::DOUBLE)<<56; - return doc->tape[json_index] == tape_double; -} -simdjson_inline bool tape_ref::is_int64() const noexcept { - constexpr uint64_t tape_int64 = uint64_t(tape_type::INT64)<<56; - return doc->tape[json_index] == tape_int64; -} -simdjson_inline bool tape_ref::is_uint64() const noexcept { - constexpr uint64_t tape_uint64 = uint64_t(tape_type::UINT64)<<56; - return doc->tape[json_index] == tape_uint64; -} -simdjson_inline bool tape_ref::is_false() const noexcept { - constexpr uint64_t tape_false = uint64_t(tape_type::FALSE_VALUE)<<56; - return doc->tape[json_index] == tape_false; -} -simdjson_inline bool tape_ref::is_true() const noexcept { - constexpr uint64_t tape_true = uint64_t(tape_type::TRUE_VALUE)<<56; - return doc->tape[json_index] == tape_true; -} -simdjson_inline bool tape_ref::is_null_on_tape() const noexcept { - constexpr uint64_t tape_null = uint64_t(tape_type::NULL_VALUE)<<56; - return doc->tape[json_index] == tape_null; -} - -inline size_t tape_ref::after_element() const noexcept { - switch (tape_ref_type()) { - case tape_type::START_ARRAY: - case tape_type::START_OBJECT: - return matching_brace_index(); - case tape_type::UINT64: - case tape_type::INT64: - case tape_type::DOUBLE: - return json_index + 2; - default: - return json_index + 1; - } -} -simdjson_inline tape_type tape_ref::tape_ref_type() const noexcept { - return static_cast(doc->tape[json_index] >> 56); -} -simdjson_inline uint64_t internal::tape_ref::tape_value() const noexcept { - return doc->tape[json_index] & internal::JSON_VALUE_MASK; -} -simdjson_inline uint32_t internal::tape_ref::matching_brace_index() const noexcept { - return uint32_t(doc->tape[json_index]); -} -simdjson_inline uint32_t internal::tape_ref::scope_count() const noexcept { - return uint32_t((doc->tape[json_index] >> 32) & internal::JSON_COUNT_MASK); -} - -template -simdjson_inline T tape_ref::next_tape_value() const noexcept { - static_assert(sizeof(T) == sizeof(uint64_t), "next_tape_value() template parameter must be 64-bit"); - // Though the following is tempting... - // return *reinterpret_cast(&doc->tape[json_index + 1]); - // It is not generally safe. It is safer, and often faster to rely - // on memcpy. Yes, it is uglier, but it is also encapsulated. - T x; - std::memcpy(&x,&doc->tape[json_index + 1],sizeof(uint64_t)); - return x; -} - -simdjson_inline uint32_t internal::tape_ref::get_string_length() const noexcept { - size_t string_buf_index = size_t(tape_value()); - uint32_t len; - std::memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len)); - return len; -} - -simdjson_inline const char * internal::tape_ref::get_c_str() const noexcept { - size_t string_buf_index = size_t(tape_value()); - return reinterpret_cast(&doc->string_buf[string_buf_index + sizeof(uint32_t)]); -} - -inline std::string_view internal::tape_ref::get_string_view() const noexcept { - return std::string_view( - get_c_str(), - get_string_length() - ); -} - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_TAPE_REF_INL_H -/* end file simdjson/internal/tape_ref-inl.h */ - -#include - -namespace simdjson { - -// -// simdjson_result inline implementation -// -simdjson_inline simdjson_result::simdjson_result() noexcept - : internal::simdjson_result_base() {} -simdjson_inline simdjson_result::simdjson_result(dom::array value) noexcept - : internal::simdjson_result_base(std::forward(value)) {} -simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept - : internal::simdjson_result_base(error) {} - -#if SIMDJSON_EXCEPTIONS - -inline dom::array::iterator simdjson_result::begin() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.begin(); -} -inline dom::array::iterator simdjson_result::end() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.end(); -} -inline size_t simdjson_result::size() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.size(); -} - -#endif // SIMDJSON_EXCEPTIONS - -inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) const noexcept { - if (error()) { return error(); } - return first.at_pointer(json_pointer); -} - - inline simdjson_result simdjson_result::at_path(std::string_view json_path) const noexcept { - auto json_pointer = json_path_to_pointer_conversion(json_path); - if (json_pointer == "-1") { return INVALID_JSON_POINTER; } - return at_pointer(json_pointer); - } - -inline simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) const noexcept { - if (error()) { - return error(); - } - return first.at_path_with_wildcard(json_path); -} - -inline simdjson_result simdjson_result::at(size_t index) const noexcept { - if (error()) { return error(); } - return first.at(index); -} - -inline std::vector& simdjson_result::get_values(std::vector& out) const noexcept { - return first.get_values(out); -} - -namespace dom { - -// -// array inline implementation -// -simdjson_inline array::array() noexcept : tape{} {} -simdjson_inline array::array(const internal::tape_ref &_tape) noexcept : tape{_tape} {} -inline array::iterator array::begin() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return internal::tape_ref(tape.doc, tape.json_index + 1); -} -inline array::iterator array::end() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return internal::tape_ref(tape.doc, tape.after_element() - 1); -} -inline size_t array::size() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return tape.scope_count(); -} -inline size_t array::number_of_slots() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return tape.matching_brace_index() - tape.json_index; -} -inline simdjson_result array::at_pointer(std::string_view json_pointer) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - if(json_pointer.empty()) { // an empty string means that we return the current node - return element(this->tape); // copy the current node - } else if(json_pointer[0] != '/') { // otherwise there is an error - return INVALID_JSON_POINTER; - } - json_pointer = json_pointer.substr(1); - // - means "the append position" or "the element after the end of the array" - // We don't support this, because we're returning a real element, not a position. - if (json_pointer == "-") { return INDEX_OUT_OF_BOUNDS; } - - // Read the array index - size_t array_index = 0; - size_t i; - for (i = 0; i < json_pointer.length() && json_pointer[i] != '/'; i++) { - uint8_t digit = uint8_t(json_pointer[i] - '0'); - // Check for non-digit in array index. If it's there, we're trying to get a field in an object - if (digit > 9) { return INCORRECT_TYPE; } - array_index = array_index*10 + digit; - } - - // 0 followed by other digits is invalid - if (i > 1 && json_pointer[0] == '0') { return INVALID_JSON_POINTER; } // "JSON pointer array index has other characters after 0" - - // Empty string is invalid; so is a "/" with no digits before it - if (i == 0) { return INVALID_JSON_POINTER; } // "Empty string in JSON pointer array index" - - // Get the child - auto child = array(tape).at(array_index); - // If there is an error, it ends here - if(child.error()) { - return child; - } - // If there is a /, we're not done yet, call recursively. - if (i < json_pointer.length()) { - child = child.at_pointer(json_pointer.substr(i)); - } - return child; -} - -inline simdjson_result array::at_path(std::string_view json_path) const noexcept { - auto json_pointer = json_path_to_pointer_conversion(json_path); - if (json_pointer == "-1") { return INVALID_JSON_POINTER; } - return at_pointer(json_pointer); -} - -inline void array::process_json_path_of_child_elements(std::vector::iterator& current, std::vector::iterator& end, const std::string_view& path_suffix, std::vector& accumulator) const noexcept { - if (current == end) { - return; - } - - simdjson_result> result; - - - for (auto it = current; it != end; ++it) { - std::vector child_result; - auto error = it->at_path_with_wildcard(path_suffix).get(child_result); - if(error) { - continue; - } - accumulator.reserve(accumulator.size() + child_result.size()); - accumulator.insert(accumulator.end(), - std::make_move_iterator(child_result.begin()), - std::make_move_iterator(child_result.end())); - } -} - -inline simdjson_result> array::at_path_with_wildcard(std::string_view json_path) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - - size_t i = 0; - // json_path.starts_with('$') requires C++20. - if (!json_path.empty() && json_path.front() == '$') { - i = 1; - } - - if (i >= json_path.size() || (json_path[i] != '.' && json_path[i] != '[')) { - return INVALID_JSON_POINTER; - } - - if (json_path.find("*") != std::string::npos) { - std::vector child_values; - - if ( - (json_path.compare(i, 3, "[*]") == 0 && json_path.size() == i + 3) || - (json_path.compare(i, 2,".*") == 0 && json_path.size() == i + 2) - ) { - get_values(child_values); - return child_values; - } - - std::pair key_and_json_path = get_next_key_and_json_path(json_path); - - std::string_view key = key_and_json_path.first; - json_path = key_and_json_path.second; - - if (key.size() > 0) { - if (key == "*") { - get_values(child_values); - } else { - element pointer_result; - std::string json_pointer = std::string("/") + std::string(key); - auto error = at_pointer(json_pointer).get(pointer_result); - - if (!error) { - child_values.emplace_back(pointer_result); - } - } - - std::vector result = {}; - - if (child_values.size() > 0) { - std::vector::iterator child_values_begin = child_values.begin(); - std::vector::iterator child_values_end = child_values.end(); - - process_json_path_of_child_elements(child_values_begin, child_values_end, json_path, result); - } - - return result; - } else { - return INVALID_JSON_POINTER; - } - } else { - element result; - auto error = at_path(json_path).get(result); - if (error) { - return error; - } - - return std::vector{std::move(result)}; - } -} - -inline simdjson_result array::at(size_t index) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - size_t i=0; - for (auto element : *this) { - if (i == index) { return element; } - i++; - } - return INDEX_OUT_OF_BOUNDS; -} - -inline std::vector& array::get_values(std::vector& out) const noexcept { - out.reserve(this->size()); - for (auto element : *this) { - out.emplace_back(element); - } - - return out; -} - -inline array::operator element() const noexcept { - return element(tape); -} - -// -// array::iterator inline implementation -// -simdjson_inline array::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } -inline element array::iterator::operator*() const noexcept { - return element(tape); -} -inline array::iterator& array::iterator::operator++() noexcept { - tape.json_index = tape.after_element(); - return *this; -} -inline array::iterator array::iterator::operator++(int) noexcept { - array::iterator out = *this; - ++*this; - return out; -} -inline bool array::iterator::operator!=(const array::iterator& other) const noexcept { - return tape.json_index != other.tape.json_index; -} -inline bool array::iterator::operator==(const array::iterator& other) const noexcept { - return tape.json_index == other.tape.json_index; -} -inline bool array::iterator::operator<(const array::iterator& other) const noexcept { - return tape.json_index < other.tape.json_index; -} -inline bool array::iterator::operator<=(const array::iterator& other) const noexcept { - return tape.json_index <= other.tape.json_index; -} -inline bool array::iterator::operator>=(const array::iterator& other) const noexcept { - return tape.json_index >= other.tape.json_index; -} -inline bool array::iterator::operator>(const array::iterator& other) const noexcept { - return tape.json_index > other.tape.json_index; -} - -} // namespace dom - - -} // namespace simdjson - -/* including simdjson/dom/element-inl.h: #include "simdjson/dom/element-inl.h" */ -/* begin file simdjson/dom/element-inl.h */ -#ifndef SIMDJSON_ELEMENT_INL_H -#define SIMDJSON_ELEMENT_INL_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/element.h" */ -/* skipped duplicate #include "simdjson/dom/document.h" */ -/* skipped duplicate #include "simdjson/dom/object.h" */ -/* skipped duplicate #include "simdjson/internal/tape_type.h" */ - -/* including simdjson/dom/object-inl.h: #include "simdjson/dom/object-inl.h" */ -/* begin file simdjson/dom/object-inl.h */ -#ifndef SIMDJSON_OBJECT_INL_H -#define SIMDJSON_OBJECT_INL_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/object.h" */ -/* skipped duplicate #include "simdjson/dom/document.h" */ - -/* skipped duplicate #include "simdjson/dom/element-inl.h" */ -/* skipped duplicate #include "simdjson/error-inl.h" */ -/* skipped duplicate #include "simdjson/jsonpathutil.h" */ - -#include - -namespace simdjson { - -// -// simdjson_result inline implementation -// -simdjson_inline simdjson_result::simdjson_result() noexcept - : internal::simdjson_result_base() {} -simdjson_inline simdjson_result::simdjson_result(dom::object value) noexcept - : internal::simdjson_result_base(std::forward(value)) {} -simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept - : internal::simdjson_result_base(error) {} - -inline simdjson_result simdjson_result::operator[](std::string_view key) const noexcept { - if (error()) { return error(); } - return first[key]; -} -inline simdjson_result simdjson_result::operator[](const char *key) const noexcept { - if (error()) { return error(); } - return first[key]; -} -inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) const noexcept { - if (error()) { return error(); } - return first.at_pointer(json_pointer); -} -inline simdjson_result simdjson_result::at_path(std::string_view json_path) const noexcept { - auto json_pointer = json_path_to_pointer_conversion(json_path); - if (json_pointer == "-1") { return INVALID_JSON_POINTER; } - return at_pointer(json_pointer); -} -inline simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) const noexcept { - if (error()) { - return error(); - } - return first.at_path_with_wildcard(json_path); -} -inline simdjson_result simdjson_result::at_key(std::string_view key) const noexcept { - if (error()) { return error(); } - return first.at_key(key); -} -inline std::vector& simdjson_result::get_values(std::vector& out) const noexcept { - return first.get_values(out); -} -inline simdjson_result simdjson_result::at_key_case_insensitive(std::string_view key) const noexcept { - if (error()) { return error(); } - return first.at_key_case_insensitive(key); -} - -#if SIMDJSON_EXCEPTIONS - -inline dom::object::iterator simdjson_result::begin() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.begin(); -} -inline dom::object::iterator simdjson_result::end() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.end(); -} -inline size_t simdjson_result::size() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.size(); -} - -#endif // SIMDJSON_EXCEPTIONS - -namespace dom { - -// -// object inline implementation -// -simdjson_inline object::object() noexcept : tape{} {} -simdjson_inline object::object(const internal::tape_ref &_tape) noexcept : tape{_tape} { } -inline object::iterator object::begin() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return internal::tape_ref(tape.doc, tape.json_index + 1); -} -inline object::iterator object::end() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return internal::tape_ref(tape.doc, tape.after_element() - 1); -} -inline size_t object::size() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return tape.scope_count(); -} - -inline simdjson_result object::operator[](std::string_view key) const noexcept { - return at_key(key); -} -inline simdjson_result object::operator[](const char *key) const noexcept { - return at_key(key); -} -inline simdjson_result object::at_pointer(std::string_view json_pointer) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - if(json_pointer.empty()) { // an empty string means that we return the current node - return element(this->tape); // copy the current node - } else if(json_pointer[0] != '/') { // otherwise there is an error - return INVALID_JSON_POINTER; - } - json_pointer = json_pointer.substr(1); - size_t slash = json_pointer.find('/'); - std::string_view key = json_pointer.substr(0, slash); - // Grab the child with the given key - simdjson_result child; - - // If there is an escape character in the key, unescape it and then get the child. - size_t escape = key.find('~'); - if (escape != std::string_view::npos) { - // Unescape the key - std::string unescaped(key); - do { - switch (unescaped[escape+1]) { - case '0': - unescaped.replace(escape, 2, "~"); - break; - case '1': - unescaped.replace(escape, 2, "/"); - break; - default: - return INVALID_JSON_POINTER; // "Unexpected ~ escape character in JSON pointer"); - } - escape = unescaped.find('~', escape+1); - } while (escape != std::string::npos); - child = at_key(unescaped); - } else { - child = at_key(key); - } - if(child.error()) { - return child; // we do not continue if there was an error - } - // If there is a /, we have to recurse and look up more of the path - if (slash != std::string_view::npos) { - child = child.at_pointer(json_pointer.substr(slash)); - } - return child; -} - -inline simdjson_result object::at_path(std::string_view json_path) const noexcept { - auto json_pointer = json_path_to_pointer_conversion(json_path); - if (json_pointer == "-1") { return INVALID_JSON_POINTER; } - return at_pointer(json_pointer); -} - -inline void object::process_json_path_of_child_elements(std::vector::iterator& current, std::vector::iterator& end, const std::string_view& path_suffix, std::vector& accumulator) const noexcept { - if (current == end) { - return; - } - - simdjson_result> result; - - for (auto it = current; it != end; ++it) { - std::vector child_result; - auto error = it->at_path_with_wildcard(path_suffix).get(child_result); - if(error) { - continue; - } - accumulator.reserve(accumulator.size() + child_result.size()); - accumulator.insert(accumulator.end(), - std::make_move_iterator(child_result.begin()), - std::make_move_iterator(child_result.end())); - } -} - -inline simdjson_result> object::at_path_with_wildcard(std::string_view json_path) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - - size_t i = 0; - if (json_path.empty()) { - return INVALID_JSON_POINTER; - } - // if JSONPath starts with $, skip it - // json_path.starts_with('$') requires C++20. - if (json_path.front() == '$') { - i = 1; - } - - if (i >= json_path.size() || (json_path[i] != '.' && json_path[i] != '[')) { - // expect JSONPath expressions to always start with $ but this isn't currently - // expected in jsonpathutil.h. - return INVALID_JSON_POINTER; - } - - if (json_path.find("*") != std::string::npos) { - - std::vector child_values; - - if ( - (json_path.compare(i, 3, "[*]") == 0 && json_path.size() == i + 3) || - (json_path.compare(i, 2,".*") == 0 && json_path.size() == i + 2) - ) { - get_values(child_values); - return child_values; - } - - std::pair key_and_json_path = get_next_key_and_json_path(json_path); - - std::string_view key = key_and_json_path.first; - json_path = key_and_json_path.second; - - if (key.size() > 0) { - if (key == "*") { - get_values(child_values); - } else { - element pointer_result; - auto error = at_pointer(std::string("/") + std::string(key)).get(pointer_result); - - if (!error) { - child_values.emplace_back(pointer_result); - } - } - - std::vector result = {}; - if (child_values.size() > 0) { - - std::vector::iterator child_values_begin = child_values.begin(); - std::vector::iterator child_values_end = child_values.end(); - - process_json_path_of_child_elements(child_values_begin, child_values_end, json_path, result); - } - - return result; - } else { - return INVALID_JSON_POINTER; - } - } else { - element result; - auto error = this->at_path(json_path).get(result); - if (error) { - return error; - } - return std::vector{std::move(result)}; - } -} - -inline simdjson_result object::at_key(std::string_view key) const noexcept { - iterator end_field = end(); - for (iterator field = begin(); field != end_field; ++field) { - if (field.key_equals(key)) { - return field.value(); - } - } - return NO_SUCH_FIELD; -} - -inline std::vector& object::get_values(std::vector& out) const noexcept { - iterator end_field = end(); - iterator begin_field = begin(); - - out.reserve(std::distance(begin_field, end_field)); - for (iterator field = begin_field; field != end_field; ++field) { - out.emplace_back(field.value()); - } - - return out; -} -// In case you wonder why we need this, please see -// https://github.com/simdjson/simdjson/issues/323 -// People do seek keys in a case-insensitive manner. -inline simdjson_result object::at_key_case_insensitive(std::string_view key) const noexcept { - iterator end_field = end(); - for (iterator field = begin(); field != end_field; ++field) { - if (field.key_equals_case_insensitive(key)) { - return field.value(); - } - } - return NO_SUCH_FIELD; -} - -inline object::operator element() const noexcept { - return element(tape); -} - -// -// object::iterator inline implementation -// -simdjson_inline object::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } -inline const key_value_pair object::iterator::operator*() const noexcept { - return key_value_pair(key(), value()); -} -inline bool object::iterator::operator!=(const object::iterator& other) const noexcept { - return tape.json_index != other.tape.json_index; -} -inline bool object::iterator::operator==(const object::iterator& other) const noexcept { - return tape.json_index == other.tape.json_index; -} -inline bool object::iterator::operator<(const object::iterator& other) const noexcept { - return tape.json_index < other.tape.json_index; -} -inline bool object::iterator::operator<=(const object::iterator& other) const noexcept { - return tape.json_index <= other.tape.json_index; -} -inline bool object::iterator::operator>=(const object::iterator& other) const noexcept { - return tape.json_index >= other.tape.json_index; -} -inline bool object::iterator::operator>(const object::iterator& other) const noexcept { - return tape.json_index > other.tape.json_index; -} -inline object::iterator& object::iterator::operator++() noexcept { - tape.json_index++; - tape.json_index = tape.after_element(); - return *this; -} -inline object::iterator object::iterator::operator++(int) noexcept { - object::iterator out = *this; - ++*this; - return out; -} -inline std::string_view object::iterator::key() const noexcept { - return tape.get_string_view(); -} -inline uint32_t object::iterator::key_length() const noexcept { - return tape.get_string_length(); -} -inline const char* object::iterator::key_c_str() const noexcept { - return reinterpret_cast(&tape.doc->string_buf[size_t(tape.tape_value()) + sizeof(uint32_t)]); -} -inline element object::iterator::value() const noexcept { - return element(internal::tape_ref(tape.doc, tape.json_index + 1)); -} - -/** - * Design notes: - * Instead of constructing a string_view and then comparing it with a - * user-provided strings, it is probably more performant to have dedicated - * functions taking as a parameter the string we want to compare against - * and return true when they are equal. That avoids the creation of a temporary - * std::string_view. Though it is possible for the compiler to avoid entirely - * any overhead due to string_view, relying too much on compiler magic is - * problematic: compiler magic sometimes fail, and then what do you do? - * Also, enticing users to rely on high-performance function is probably better - * on the long run. - */ - -inline bool object::iterator::key_equals(std::string_view o) const noexcept { - // We use the fact that the key length can be computed quickly - // without access to the string buffer. - const uint32_t len = key_length(); - if(o.size() == len) { - // We avoid construction of a temporary string_view instance. - return (memcmp(o.data(), key_c_str(), len) == 0); - } - return false; -} - -inline bool object::iterator::key_equals_case_insensitive(std::string_view o) const noexcept { - // We use the fact that the key length can be computed quickly - // without access to the string buffer. - const uint32_t len = key_length(); - if(o.size() == len) { - // See For case-insensitive string comparisons, avoid char-by-char functions - // https://lemire.me/blog/2020/04/30/for-case-insensitive-string-comparisons-avoid-char-by-char-functions/ - // Note that it might be worth rolling our own strncasecmp function, with vectorization. - return (simdjson_strncasecmp(o.data(), key_c_str(), len) == 0); - } - return false; -} -// -// key_value_pair inline implementation -// -inline key_value_pair::key_value_pair(std::string_view _key, element _value) noexcept : - key(_key), value(_value) {} - -} // namespace dom - -} // namespace simdjson - -#if SIMDJSON_SUPPORTS_RANGES -static_assert(std::ranges::view); -static_assert(std::ranges::sized_range); -#if SIMDJSON_EXCEPTIONS -static_assert(std::ranges::view>); -static_assert(std::ranges::sized_range>); -#endif // SIMDJSON_EXCEPTIONS -#endif // SIMDJSON_SUPPORTS_RANGES - -#endif // SIMDJSON_OBJECT_INL_H -/* end file simdjson/dom/object-inl.h */ -/* skipped duplicate #include "simdjson/error-inl.h" */ -/* skipped duplicate #include "simdjson/jsonpathutil.h" */ - -#include -#include - -namespace simdjson { - -// -// simdjson_result inline implementation -// -simdjson_inline simdjson_result::simdjson_result() noexcept - : internal::simdjson_result_base() {} -simdjson_inline simdjson_result::simdjson_result(dom::element &&value) noexcept - : internal::simdjson_result_base(std::forward(value)) {} -simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept - : internal::simdjson_result_base(error) {} -inline simdjson_result simdjson_result::type() const noexcept { - if (error()) { return error(); } - return first.type(); -} - -template -simdjson_inline bool simdjson_result::is() const noexcept { - return !error() && first.is(); -} -template -simdjson_inline simdjson_result simdjson_result::get() const noexcept { - if (error()) { return error(); } - return first.get(); -} -template -simdjson_warn_unused simdjson_inline error_code simdjson_result::get(T &value) const noexcept { - if (error()) { return error(); } - return first.get(value); -} - -simdjson_inline simdjson_result simdjson_result::get_array() const noexcept { - if (error()) { return error(); } - return first.get_array(); -} -simdjson_inline simdjson_result simdjson_result::get_object() const noexcept { - if (error()) { return error(); } - return first.get_object(); -} -simdjson_inline simdjson_result simdjson_result::get_c_str() const noexcept { - if (error()) { return error(); } - return first.get_c_str(); -} -simdjson_inline simdjson_result simdjson_result::get_string_length() const noexcept { - if (error()) { return error(); } - return first.get_string_length(); -} -simdjson_inline simdjson_result simdjson_result::get_string() const noexcept { - if (error()) { return error(); } - return first.get_string(); -} -simdjson_inline simdjson_result simdjson_result::get_int64() const noexcept { - if (error()) { return error(); } - return first.get_int64(); -} -simdjson_inline simdjson_result simdjson_result::get_uint64() const noexcept { - if (error()) { return error(); } - return first.get_uint64(); -} -simdjson_inline simdjson_result simdjson_result::get_double() const noexcept { - if (error()) { return error(); } - return first.get_double(); -} -simdjson_inline simdjson_result simdjson_result::get_bool() const noexcept { - if (error()) { return error(); } - return first.get_bool(); -} -simdjson_inline simdjson_result simdjson_result::get_bigint() const noexcept { - if (error()) { return error(); } - return first.get_bigint(); -} - -simdjson_inline bool simdjson_result::is_array() const noexcept { - return !error() && first.is_array(); -} -simdjson_inline bool simdjson_result::is_object() const noexcept { - return !error() && first.is_object(); -} -simdjson_inline bool simdjson_result::is_string() const noexcept { - return !error() && first.is_string(); -} -simdjson_inline bool simdjson_result::is_int64() const noexcept { - return !error() && first.is_int64(); -} -simdjson_inline bool simdjson_result::is_uint64() const noexcept { - return !error() && first.is_uint64(); -} -simdjson_inline bool simdjson_result::is_double() const noexcept { - return !error() && first.is_double(); -} -simdjson_inline bool simdjson_result::is_number() const noexcept { - return !error() && first.is_number(); -} -simdjson_inline bool simdjson_result::is_bool() const noexcept { - return !error() && first.is_bool(); -} - -simdjson_inline bool simdjson_result::is_null() const noexcept { - return !error() && first.is_null(); -} -simdjson_inline bool simdjson_result::is_bigint() const noexcept { - return !error() && first.is_bigint(); -} - -simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) const noexcept { - if (error()) { return error(); } - return first[key]; -} -simdjson_inline simdjson_result simdjson_result::operator[](const char *key) const noexcept { - if (error()) { return error(); } - return first[key]; -} -simdjson_inline simdjson_result simdjson_result::at_pointer(const std::string_view json_pointer) const noexcept { - if (error()) { return error(); } - return first.at_pointer(json_pointer); -} -simdjson_inline simdjson_result simdjson_result::at_path(const std::string_view json_path) const noexcept { - auto json_pointer = json_path_to_pointer_conversion(json_path); - if (json_pointer == "-1") { return INVALID_JSON_POINTER; } - return at_pointer(json_pointer); -} - -simdjson_inline simdjson_result> simdjson_result::at_path_with_wildcard(const std::string_view json_path) const noexcept { - if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); -} - -#ifndef SIMDJSON_DISABLE_DEPRECATED_API -[[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] -simdjson_inline simdjson_result simdjson_result::at(const std::string_view json_pointer) const noexcept { -SIMDJSON_PUSH_DISABLE_WARNINGS -SIMDJSON_DISABLE_DEPRECATED_WARNING - if (error()) { return error(); } - return first.at(json_pointer); -SIMDJSON_POP_DISABLE_WARNINGS -} -#endif // SIMDJSON_DISABLE_DEPRECATED_API -simdjson_inline simdjson_result simdjson_result::at(size_t index) const noexcept { - if (error()) { return error(); } - return first.at(index); -} -simdjson_inline simdjson_result simdjson_result::at_key(std::string_view key) const noexcept { - if (error()) { return error(); } - return first.at_key(key); -} -simdjson_inline simdjson_result simdjson_result::at_key_case_insensitive(std::string_view key) const noexcept { - if (error()) { return error(); } - return first.at_key_case_insensitive(key); -} - -#if SIMDJSON_EXCEPTIONS - -simdjson_inline simdjson_result::operator bool() const noexcept(false) { - return get(); -} -simdjson_inline simdjson_result::operator const char *() const noexcept(false) { - return get(); -} -simdjson_inline simdjson_result::operator std::string_view() const noexcept(false) { - return get(); -} -simdjson_inline simdjson_result::operator uint64_t() const noexcept(false) { - return get(); -} -simdjson_inline simdjson_result::operator int64_t() const noexcept(false) { - return get(); -} -simdjson_inline simdjson_result::operator double() const noexcept(false) { - return get(); -} -simdjson_inline simdjson_result::operator dom::array() const noexcept(false) { - return get(); -} -simdjson_inline simdjson_result::operator dom::object() const noexcept(false) { - return get(); -} - -simdjson_inline dom::array::iterator simdjson_result::begin() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.begin(); -} -simdjson_inline dom::array::iterator simdjson_result::end() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.end(); -} - -#endif // SIMDJSON_EXCEPTIONS - -namespace dom { - -// -// element inline implementation -// -simdjson_inline element::element() noexcept : tape{} {} -simdjson_inline element::element(const internal::tape_ref &_tape) noexcept : tape{_tape} { } - -inline element_type element::type() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - auto tape_type = tape.tape_ref_type(); - return tape_type == internal::tape_type::FALSE_VALUE ? element_type::BOOL : static_cast(tape_type); -} - -inline simdjson_result element::get_bool() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - if(tape.is_true()) { - return true; - } else if(tape.is_false()) { - return false; - } - return INCORRECT_TYPE; -} -inline simdjson_result element::get_bigint() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); - switch (tape.tape_ref_type()) { - case internal::tape_type::BIGINT: - return tape.get_string_view(); - default: - return INCORRECT_TYPE; - } -} -inline simdjson_result element::get_c_str() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - switch (tape.tape_ref_type()) { - case internal::tape_type::STRING: { - return tape.get_c_str(); - } - default: - return INCORRECT_TYPE; - } -} -inline simdjson_result element::get_string_length() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - switch (tape.tape_ref_type()) { - case internal::tape_type::STRING: { - return tape.get_string_length(); - } - default: - return INCORRECT_TYPE; - } -} -inline simdjson_result element::get_string() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - switch (tape.tape_ref_type()) { - case internal::tape_type::STRING: - return tape.get_string_view(); - default: - return INCORRECT_TYPE; - } -} -inline simdjson_result element::get_uint64() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - if(simdjson_unlikely(!tape.is_uint64())) { // branch rarely taken - if(tape.is_int64()) { - int64_t result = tape.next_tape_value(); - if (result < 0) { - return NUMBER_OUT_OF_RANGE; - } - return uint64_t(result); - } - return INCORRECT_TYPE; - } - return tape.next_tape_value(); -} -inline simdjson_result element::get_int64() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - if(simdjson_unlikely(!tape.is_int64())) { // branch rarely taken - if(tape.is_uint64()) { - uint64_t result = tape.next_tape_value(); - // Wrapping max in parens to handle Windows issue: https://stackoverflow.com/questions/11544073/how-do-i-deal-with-the-max-macro-in-windows-h-colliding-with-max-in-std - if (result > uint64_t((std::numeric_limits::max)())) { - return NUMBER_OUT_OF_RANGE; - } - return static_cast(result); - } - return INCORRECT_TYPE; - } - return tape.next_tape_value(); -} -inline simdjson_result element::get_double() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - // Performance considerations: - // 1. Querying tape_ref_type() implies doing a shift, it is fast to just do a straight - // comparison. - // 2. Using a switch-case relies on the compiler guessing what kind of code generation - // we want... But the compiler cannot know that we expect the type to be "double" - // most of the time. - // We can expect get to refer to a double type almost all the time. - // It is important to craft the code accordingly so that the compiler can use this - // information. (This could also be solved with profile-guided optimization.) - if(simdjson_unlikely(!tape.is_double())) { // branch rarely taken - if(tape.is_uint64()) { - return double(tape.next_tape_value()); - } else if(tape.is_int64()) { - return double(tape.next_tape_value()); - } - return INCORRECT_TYPE; - } - // this is common: - return tape.next_tape_value(); -} -inline simdjson_result element::get_array() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - switch (tape.tape_ref_type()) { - case internal::tape_type::START_ARRAY: - return array(tape); - default: - return INCORRECT_TYPE; - } -} -inline simdjson_result element::get_object() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - switch (tape.tape_ref_type()) { - case internal::tape_type::START_OBJECT: - return object(tape); - default: - return INCORRECT_TYPE; - } -} - -template -simdjson_warn_unused simdjson_inline error_code element::get(T &value) const noexcept { - return get().get(value); -} -// An element-specific version prevents recursion with simdjson_result::get(value) -template<> -simdjson_warn_unused simdjson_inline error_code element::get(element &value) const noexcept { - value = element(tape); - return SUCCESS; -} -template -inline void element::tie(T &value, error_code &error) && noexcept { - error = get(value); -} - -template -simdjson_inline bool element::is() const noexcept { - auto result = get(); - return !result.error(); -} - -template<> inline simdjson_result element::get() const noexcept { return get_array(); } -template<> inline simdjson_result element::get() const noexcept { return get_object(); } -template<> inline simdjson_result element::get() const noexcept { return get_c_str(); } -template<> inline simdjson_result element::get() const noexcept { return get_string(); } -template<> inline simdjson_result element::get() const noexcept { return get_int64(); } -template<> inline simdjson_result element::get() const noexcept { return get_uint64(); } -template<> inline simdjson_result element::get() const noexcept { return get_double(); } -template<> inline simdjson_result element::get() const noexcept { return get_bool(); } - -inline bool element::is_array() const noexcept { return is(); } -inline bool element::is_object() const noexcept { return is(); } -inline bool element::is_string() const noexcept { return is(); } -inline bool element::is_int64() const noexcept { return is(); } -inline bool element::is_uint64() const noexcept { return is(); } -inline bool element::is_double() const noexcept { return is(); } -inline bool element::is_bool() const noexcept { return is(); } -inline bool element::is_number() const noexcept { return is_int64() || is_uint64() || is_double(); } - -inline bool element::is_null() const noexcept { - return tape.is_null_on_tape(); -} - -inline bool element::is_bigint() const noexcept { - return tape.tape_ref_type() == internal::tape_type::BIGINT; -} - -#if SIMDJSON_EXCEPTIONS - -inline element::operator bool() const noexcept(false) { return get(); } -inline element::operator const char*() const noexcept(false) { return get(); } -inline element::operator std::string_view() const noexcept(false) { return get(); } -inline element::operator uint64_t() const noexcept(false) { return get(); } -inline element::operator int64_t() const noexcept(false) { return get(); } -inline element::operator double() const noexcept(false) { return get(); } -inline element::operator array() const noexcept(false) { return get(); } -inline element::operator object() const noexcept(false) { return get(); } - -inline array::iterator element::begin() const noexcept(false) { - return get().begin(); -} -inline array::iterator element::end() const noexcept(false) { - return get().end(); -} - -#endif // SIMDJSON_EXCEPTIONS - -inline simdjson_result element::operator[](std::string_view key) const noexcept { - return at_key(key); -} -inline simdjson_result element::operator[](const char *key) const noexcept { - return at_key(key); -} - -inline bool is_pointer_well_formed(std::string_view json_pointer) noexcept { - if (simdjson_unlikely(json_pointer[0] != '/')) { - return false; - } - size_t escape = json_pointer.find('~'); - if (escape == std::string_view::npos) { - return true; - } - if (escape == json_pointer.size() - 1) { - return false; - } - if (json_pointer[escape + 1] != '0' && json_pointer[escape + 1] != '1') { - return false; - } - return true; -} - -inline simdjson_result element::at_pointer(std::string_view json_pointer) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - switch (tape.tape_ref_type()) { - case internal::tape_type::START_OBJECT: - return object(tape).at_pointer(json_pointer); - case internal::tape_type::START_ARRAY: - return array(tape).at_pointer(json_pointer); - default: { - if (!json_pointer.empty()) { // a non-empty string can be invalid, or accessing a primitive (issue 2154) - if (is_pointer_well_formed(json_pointer)) { - return NO_SUCH_FIELD; - } - return INVALID_JSON_POINTER; - } - // an empty string means that we return the current node - dom::element copy(*this); - return simdjson_result(std::move(copy)); - } - } -} - -inline simdjson_result> element::at_path_with_wildcard(std::string_view json_path) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - - switch (tape.tape_ref_type()) { - case internal::tape_type::START_OBJECT: - return object(tape).at_path_with_wildcard(json_path); - case internal::tape_type::START_ARRAY: - return array(tape).at_path_with_wildcard(json_path); - default: - return std::vector{}; - } -} - -inline simdjson_result element::at_path(std::string_view json_path) const noexcept { - auto json_pointer = json_path_to_pointer_conversion(json_path); - if (json_pointer == "-1") { return INVALID_JSON_POINTER; } - return at_pointer(json_pointer); -} -#ifndef SIMDJSON_DISABLE_DEPRECATED_API -[[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] -inline simdjson_result element::at(std::string_view json_pointer) const noexcept { - // version 0.4 of simdjson allowed non-compliant pointers - auto std_pointer = (json_pointer.empty() ? "" : "/") + std::string(json_pointer.begin(), json_pointer.end()); - return at_pointer(std_pointer); -} -#endif // SIMDJSON_DISABLE_DEPRECATED_API - -inline simdjson_result element::at(size_t index) const noexcept { - return get().at(index); -} -inline simdjson_result element::at_key(std::string_view key) const noexcept { - return get().at_key(key); -} -inline simdjson_result element::at_key_case_insensitive(std::string_view key) const noexcept { - return get().at_key_case_insensitive(key); -} -inline bool element::operator<(const element &other) const noexcept { - return tape.json_index < other.tape.json_index; -} -inline bool element::operator==(const element &other) const noexcept { - return tape.json_index == other.tape.json_index; -} - -inline bool element::dump_raw_tape(std::ostream &out) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return tape.doc->dump_raw_tape(out); -} - - -inline std::ostream& operator<<(std::ostream& out, element_type type) { - switch (type) { - case element_type::ARRAY: - return out << "array"; - case element_type::OBJECT: - return out << "object"; - case element_type::INT64: - return out << "int64_t"; - case element_type::UINT64: - return out << "uint64_t"; - case element_type::DOUBLE: - return out << "double"; - case element_type::STRING: - return out << "string"; - case element_type::BOOL: - return out << "bool"; - case element_type::NULL_VALUE: - return out << "null"; - case element_type::BIGINT: - return out << "bigint"; - default: - return out << "unexpected content!!!"; // abort() usage is forbidden in the library - } -} - -} // namespace dom - -} // namespace simdjson - -#endif // SIMDJSON_ELEMENT_INL_H -/* end file simdjson/dom/element-inl.h */ - -#if SIMDJSON_SUPPORTS_RANGES -static_assert(std::ranges::view); -static_assert(std::ranges::sized_range); -#if SIMDJSON_EXCEPTIONS -static_assert(std::ranges::view>); -static_assert(std::ranges::sized_range>); -#endif // SIMDJSON_EXCEPTIONS -#endif // SIMDJSON_SUPPORTS_RANGES - -#endif // SIMDJSON_ARRAY_INL_H -/* end file simdjson/dom/array-inl.h */ -/* including simdjson/dom/document_stream-inl.h: #include "simdjson/dom/document_stream-inl.h" */ -/* begin file simdjson/dom/document_stream-inl.h */ -#ifndef SIMDJSON_DOCUMENT_STREAM_INL_H -#define SIMDJSON_DOCUMENT_STREAM_INL_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/document_stream.h" */ -/* skipped duplicate #include "simdjson/dom/element-inl.h" */ -/* including simdjson/dom/parser-inl.h: #include "simdjson/dom/parser-inl.h" */ -/* begin file simdjson/dom/parser-inl.h */ -#ifndef SIMDJSON_PARSER_INL_H -#define SIMDJSON_PARSER_INL_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/document_stream.h" */ -/* skipped duplicate #include "simdjson/implementation.h" */ -/* skipped duplicate #include "simdjson/internal/dom_parser_implementation.h" */ - -/* skipped duplicate #include "simdjson/error-inl.h" */ -/* skipped duplicate #include "simdjson/padded_string-inl.h" */ -/* skipped duplicate #include "simdjson/dom/document_stream-inl.h" */ -/* skipped duplicate #include "simdjson/dom/element-inl.h" */ - -#include -#include /* memcmp */ - -namespace simdjson { -namespace dom { - -// -// parser inline implementation -// -simdjson_inline parser::parser(size_t max_capacity) noexcept - : _max_capacity{max_capacity}, - loaded_bytes(nullptr) { -} -simdjson_inline parser::parser(parser &&other) noexcept = default; -simdjson_inline parser &parser::operator=(parser &&other) noexcept = default; - -inline bool parser::is_valid() const noexcept { return valid; } -inline int parser::get_error_code() const noexcept { return error; } -inline std::string parser::get_error_message() const noexcept { return error_message(error); } - -inline bool parser::dump_raw_tape(std::ostream &os) const noexcept { - return valid ? doc.dump_raw_tape(os) : false; -} - -inline simdjson_result parser::read_file(std::string_view path) noexcept { - const std::string path_copy(path); - // Open the file - SIMDJSON_PUSH_DISABLE_WARNINGS - SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe - std::FILE *fp = std::fopen(path_copy.c_str(), "rb"); - SIMDJSON_POP_DISABLE_WARNINGS - - if (fp == nullptr) { - return IO_ERROR; - } - - // Get the file size - int ret; -#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS - ret = _fseeki64(fp, 0, SEEK_END); -#else - ret = std::fseek(fp, 0, SEEK_END); -#endif // _WIN64 - if(ret < 0) { - std::fclose(fp); - return IO_ERROR; - } -#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS - __int64 len = _ftelli64(fp); - if(len == -1L) { - std::fclose(fp); - return IO_ERROR; - } -#else - long len = std::ftell(fp); - if((len < 0) || (len == LONG_MAX)) { - std::fclose(fp); - return IO_ERROR; - } -#endif - - // Make sure we have enough capacity to load the file - if (_loaded_bytes_capacity < size_t(len)) { - loaded_bytes.reset( internal::allocate_padded_buffer(len) ); - if (!loaded_bytes) { - std::fclose(fp); - return MEMALLOC; - } - _loaded_bytes_capacity = len; - } - - // Read the string - std::rewind(fp); - size_t bytes_read = std::fread(loaded_bytes.get(), 1, len, fp); - if (std::fclose(fp) != 0 || bytes_read != size_t(len)) { - return IO_ERROR; - } - - return bytes_read; -} - -inline simdjson_result parser::load(std::string_view path) & noexcept { - return load_into_document(doc, path); -} - -inline simdjson_result parser::load_into_document(document& provided_doc, std::string_view path) & noexcept { - size_t len; - auto _error = read_file(path).get(len); - if (_error) { return _error; } - return parse_into_document(provided_doc, loaded_bytes.get(), len, false); -} - -inline simdjson_result parser::load_many(std::string_view path, size_t batch_size) noexcept { - size_t len; - auto _error = read_file(path).get(len); - if (_error) { return _error; } - if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } - return document_stream(*this, reinterpret_cast(loaded_bytes.get()), len, batch_size); -} - -inline simdjson_result parser::parse_into_document(document& provided_doc, const uint8_t *buf, size_t len, bool realloc_if_needed) & noexcept { - // Important: we need to ensure that document has enough capacity. - // Important: It is possible that provided_doc is actually the internal 'doc' within the parser!!! - error_code _error = ensure_capacity(provided_doc, len); - if (_error) { return _error; } - if (realloc_if_needed) { - // Make sure we have enough capacity to copy len bytes - if (!loaded_bytes || _loaded_bytes_capacity < len) { - loaded_bytes.reset( internal::allocate_padded_buffer(len) ); - if (!loaded_bytes) { - return MEMALLOC; - } - _loaded_bytes_capacity = len; - } - std::memcpy(static_cast(loaded_bytes.get()), buf, len); - buf = reinterpret_cast(loaded_bytes.get()); - } - - if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { - buf += 3; - len -= 3; - } - implementation->_number_as_string = _number_as_string; - _error = implementation->parse(buf, len, provided_doc); - - if (_error) { return _error; } - - return provided_doc.root(); -} - -simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const char *buf, size_t len, bool realloc_if_needed) & noexcept { - return parse_into_document(provided_doc, reinterpret_cast(buf), len, realloc_if_needed); -} -simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const std::string &s) & noexcept { - return parse_into_document(provided_doc, s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); -} -simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const padded_string &s) & noexcept { - return parse_into_document(provided_doc, s.data(), s.length(), false); -} - - -inline simdjson_result parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) & noexcept { - return parse_into_document(doc, buf, len, realloc_if_needed); -} - -simdjson_inline simdjson_result parser::parse(const char *buf, size_t len, bool realloc_if_needed) & noexcept { - return parse(reinterpret_cast(buf), len, realloc_if_needed); -} -simdjson_inline simdjson_result parser::parse(const std::string &s) & noexcept { - return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); -} -simdjson_inline simdjson_result parser::parse(const padded_string &s) & noexcept { - return parse(s.data(), s.length(), false); -} -simdjson_inline simdjson_result parser::parse(const padded_string_view &v) & noexcept { - return parse(v.data(), v.length(), false); -} - -inline simdjson_result parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept { - if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } - if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { - buf += 3; - len -= 3; - } - return document_stream(*this, buf, len, batch_size); -} -inline simdjson_result parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept { - return parse_many(reinterpret_cast(buf), len, batch_size); -} -inline simdjson_result parser::parse_many(const std::string &s, size_t batch_size) noexcept { - return parse_many(s.data(), s.length(), batch_size); -} -inline simdjson_result parser::parse_many(const padded_string &s, size_t batch_size) noexcept { - return parse_many(s.data(), s.length(), batch_size); -} - -simdjson_inline size_t parser::capacity() const noexcept { - return implementation ? implementation->capacity() : 0; -} -simdjson_inline size_t parser::max_capacity() const noexcept { - return _max_capacity; -} -simdjson_pure simdjson_inline size_t parser::max_depth() const noexcept { - return implementation ? implementation->max_depth() : DEFAULT_MAX_DEPTH; -} - -simdjson_warn_unused -inline error_code parser::allocate(size_t capacity, size_t max_depth) noexcept { - // - // Reallocate implementation if needed - // - error_code err; - if (implementation) { - err = implementation->allocate(capacity, max_depth); - } else { - err = simdjson::get_active_implementation()->create_dom_parser_implementation(capacity, max_depth, implementation); - } - if (err) { return err; } - return SUCCESS; -} - -#ifndef SIMDJSON_DISABLE_DEPRECATED_API -simdjson_warn_unused -inline bool parser::allocate_capacity(size_t capacity, size_t max_depth) noexcept { - return !allocate(capacity, max_depth); -} -#endif // SIMDJSON_DISABLE_DEPRECATED_API - -inline error_code parser::ensure_capacity(size_t desired_capacity) noexcept { - return ensure_capacity(doc, desired_capacity); -} - - -inline error_code parser::ensure_capacity(document& target_document, size_t desired_capacity) noexcept { - // 1. It is wasteful to allocate a document and a parser for documents spanning less than MINIMAL_DOCUMENT_CAPACITY bytes. - // 2. If we allow desired_capacity = 0 then it is possible to exit this function with implementation == nullptr. - if(desired_capacity < MINIMAL_DOCUMENT_CAPACITY) { desired_capacity = MINIMAL_DOCUMENT_CAPACITY; } - // If we don't have enough capacity, (try to) automatically bump it. - // If the document needs allocation, do it too. - // Both in one if statement to minimize unlikely branching. - // - // Note: we must make sure that this function is called if capacity() == 0. We do so because we - // ensure that desired_capacity > 0. - if (simdjson_unlikely(capacity() < desired_capacity || target_document.capacity() < desired_capacity)) { - if (desired_capacity > max_capacity()) { - return error = CAPACITY; - } - error_code err1 = target_document.capacity() < desired_capacity ? target_document.allocate(desired_capacity) : SUCCESS; - error_code err2 = capacity() < desired_capacity ? allocate(desired_capacity, max_depth()) : SUCCESS; - if(err1 != SUCCESS) { return error = err1; } - if(err2 != SUCCESS) { return error = err2; } - } - return SUCCESS; -} - -simdjson_inline void parser::set_max_capacity(size_t max_capacity) noexcept { - if(max_capacity > MINIMAL_DOCUMENT_CAPACITY) { - _max_capacity = max_capacity; - } else { - _max_capacity = MINIMAL_DOCUMENT_CAPACITY; - } -} - -} // namespace dom -} // namespace simdjson - -#endif // SIMDJSON_PARSER_INL_H -/* end file simdjson/dom/parser-inl.h */ -/* skipped duplicate #include "simdjson/error-inl.h" */ -/* skipped duplicate #include "simdjson/internal/dom_parser_implementation.h" */ - -namespace simdjson { -namespace dom { - -#ifdef SIMDJSON_THREADS_ENABLED - -inline void stage1_worker::finish() { - // After calling "run" someone would call finish() to wait - // for the end of the processing. - // This function will wait until either the thread has done - // the processing or, else, the destructor has been called. - std::unique_lock lock(locking_mutex); - cond_var.wait(lock, [this]{return has_work == false;}); -} - -inline stage1_worker::~stage1_worker() { - // The thread may never outlive the stage1_worker instance - // and will always be stopped/joined before the stage1_worker - // instance is gone. - stop_thread(); -} - -inline void stage1_worker::start_thread() { - std::unique_lock lock(locking_mutex); - if(thread.joinable()) { - return; // This should never happen but we never want to create more than one thread. - } - thread = std::thread([this]{ - while(true) { - std::unique_lock thread_lock(locking_mutex); - // We wait for either "run" or "stop_thread" to be called. - cond_var.wait(thread_lock, [this]{return has_work || !can_work;}); - // If, for some reason, the stop_thread() method was called (i.e., the - // destructor of stage1_worker is called, then we want to immediately destroy - // the thread (and not do any more processing). - if(!can_work) { - break; - } - this->owner->stage1_thread_error = this->owner->run_stage1(*this->stage1_thread_parser, - this->_next_batch_start); - this->has_work = false; - // The condition variable call should be moved after thread_lock.unlock() for performance - // reasons but thread sanitizers may report it as a data race if we do. - // See https://stackoverflow.com/questions/35775501/c-should-condition-variable-be-notified-under-lock - cond_var.notify_one(); // will notify "finish" - thread_lock.unlock(); - } - } - ); -} - - -inline void stage1_worker::stop_thread() { - std::unique_lock lock(locking_mutex); - // We have to make sure that all locks can be released. - can_work = false; - has_work = false; - cond_var.notify_all(); - lock.unlock(); - if(thread.joinable()) { - thread.join(); - } -} - -inline void stage1_worker::run(document_stream * ds, dom::parser * stage1, size_t next_batch_start) { - std::unique_lock lock(locking_mutex); - owner = ds; - _next_batch_start = next_batch_start; - stage1_thread_parser = stage1; - has_work = true; - // The condition variable call should be moved after thread_lock.unlock() for performance - // reasons but thread sanitizers may report it as a data race if we do. - // See https://stackoverflow.com/questions/35775501/c-should-condition-variable-be-notified-under-lock - cond_var.notify_one(); // will notify the thread lock that we have work - lock.unlock(); -} -#endif - -simdjson_inline document_stream::document_stream( - dom::parser &_parser, - const uint8_t *_buf, - size_t _len, - size_t _batch_size -) noexcept - : parser{&_parser}, - buf{_buf}, - len{_len}, - batch_size{_batch_size <= MINIMAL_BATCH_SIZE ? MINIMAL_BATCH_SIZE : _batch_size}, - error{SUCCESS} -#ifdef SIMDJSON_THREADS_ENABLED - , use_thread(_parser.threaded) // we need to make a copy because _parser.threaded can change -#endif -{ -#ifdef SIMDJSON_THREADS_ENABLED - if(worker.get() == nullptr) { - error = MEMALLOC; - } -#endif -} - -simdjson_inline document_stream::document_stream() noexcept - : parser{nullptr}, - buf{nullptr}, - len{0}, - batch_size{0}, - error{UNINITIALIZED} -#ifdef SIMDJSON_THREADS_ENABLED - , use_thread(false) -#endif -{ -} - -simdjson_inline document_stream::~document_stream() noexcept { -#ifdef SIMDJSON_THREADS_ENABLED - worker.reset(); -#endif -} - -simdjson_inline document_stream::iterator::iterator() noexcept - : stream{nullptr}, finished{true} { -} - -simdjson_inline document_stream::iterator document_stream::begin() noexcept { - start(); - // If there are no documents, we're finished. - return iterator(this, error == EMPTY); -} - -simdjson_inline document_stream::iterator document_stream::end() noexcept { - return iterator(this, true); -} - -simdjson_inline document_stream::iterator::iterator(document_stream* _stream, bool is_end) noexcept - : stream{_stream}, finished{is_end} { -} - -simdjson_inline document_stream::iterator::reference document_stream::iterator::operator*() noexcept { - // Note that in case of error, we do not yet mark - // the iterator as "finished": this detection is done - // in the operator++ function since it is possible - // to call operator++ repeatedly while omitting - // calls to operator*. - if (stream->error) { return stream->error; } - return stream->parser->doc.root(); -} - -simdjson_inline document_stream::iterator& document_stream::iterator::operator++() noexcept { - // If there is an error, then we want the iterator - // to be finished, no matter what. (E.g., we do not - // keep generating documents with errors, or go beyond - // a document with errors.) - // - // Users do not have to call "operator*()" when they use operator++, - // so we need to end the stream in the operator++ function. - // - // Note that setting finished = true is essential otherwise - // we would enter an infinite loop. - if (stream->error) { finished = true; } - // Note that stream->error() is guarded against error conditions - // (it will immediately return if stream->error casts to false). - // In effect, this next function does nothing when (stream->error) - // is true (hence the risk of an infinite loop). - stream->next(); - // If that was the last document, we're finished. - // It is the only type of error we do not want to appear - // in operator*. - if (stream->error == EMPTY) { finished = true; } - // If we had any other kind of error (not EMPTY) then we want - // to pass it along to the operator* and we cannot mark the result - // as "finished" just yet. - return *this; -} - -simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { - return finished != other.finished; -} - -inline void document_stream::start() noexcept { - if (error) { return; } - error = parser->ensure_capacity(batch_size); - if (error) { return; } - // Always run the first stage 1 parse immediately - batch_start = 0; - error = run_stage1(*parser, batch_start); - while(error == EMPTY) { - // In exceptional cases, we may start with an empty block - batch_start = next_batch_start(); - if (batch_start >= len) { return; } - error = run_stage1(*parser, batch_start); - } - if (error) { return; } -#ifdef SIMDJSON_THREADS_ENABLED - if (use_thread && next_batch_start() < len) { - // Kick off the first thread if needed - error = stage1_thread_parser.ensure_capacity(batch_size); - if (error) { return; } - worker->start_thread(); - start_stage1_thread(); - if (error) { return; } - } -#endif // SIMDJSON_THREADS_ENABLED - next(); -} - -simdjson_inline size_t document_stream::iterator::current_index() const noexcept { - return stream->doc_index; -} - -simdjson_inline std::string_view document_stream::iterator::source() const noexcept { - const char* start = reinterpret_cast(stream->buf) + current_index(); - bool object_or_array = ((*start == '[') || (*start == '{')); - if(object_or_array) { - size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index - 1]; - return std::string_view(start, next_doc_index - current_index() + 1); - } else { - size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index]; - size_t svlen = next_doc_index - current_index(); - while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0')) { - svlen--; - } - return std::string_view(start, svlen); - } -} - - -inline void document_stream::next() noexcept { - // We always exit at once, once in an error condition. - if (error) { return; } - - // Load the next document from the batch - doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index]; - error = parser->implementation->stage2_next(parser->doc); - // If that was the last document in the batch, load another batch (if available) - while (error == EMPTY) { - batch_start = next_batch_start(); - if (batch_start >= len) { break; } - -#ifdef SIMDJSON_THREADS_ENABLED - if(use_thread) { - load_from_stage1_thread(); - } else { - error = run_stage1(*parser, batch_start); - } -#else - error = run_stage1(*parser, batch_start); -#endif - if (error) { continue; } // If the error was EMPTY, we may want to load another batch. - // Run stage 2 on the first document in the batch - doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index]; - error = parser->implementation->stage2_next(parser->doc); - } -} -inline size_t document_stream::size_in_bytes() const noexcept { - return len; -} - -inline size_t document_stream::truncated_bytes() const noexcept { - if(error == CAPACITY) { return len - batch_start; } - return parser->implementation->structural_indexes[parser->implementation->n_structural_indexes] - parser->implementation->structural_indexes[parser->implementation->n_structural_indexes + 1]; -} - -inline size_t document_stream::next_batch_start() const noexcept { - return batch_start + parser->implementation->structural_indexes[parser->implementation->n_structural_indexes]; -} - -inline error_code document_stream::run_stage1(dom::parser &p, size_t _batch_start) noexcept { - size_t remaining = len - _batch_start; - if (remaining <= batch_size) { - return p.implementation->stage1(&buf[_batch_start], remaining, stage1_mode::streaming_final); - } else { - return p.implementation->stage1(&buf[_batch_start], batch_size, stage1_mode::streaming_partial); - } -} - -#ifdef SIMDJSON_THREADS_ENABLED - -inline void document_stream::load_from_stage1_thread() noexcept { - worker->finish(); - // Swap to the parser that was loaded up in the thread. Make sure the parser has - // enough memory to swap to, as well. - std::swap(*parser, stage1_thread_parser); - error = stage1_thread_error; - if (error) { return; } - - // If there's anything left, start the stage 1 thread! - if (next_batch_start() < len) { - start_stage1_thread(); - } -} - -inline void document_stream::start_stage1_thread() noexcept { - // we call the thread on a lambda that will update - // this->stage1_thread_error - // there is only one thread that may write to this value - // TODO this is NOT exception-safe. - this->stage1_thread_error = UNINITIALIZED; // In case something goes wrong, make sure it's an error - size_t _next_batch_start = this->next_batch_start(); - - worker->run(this, & this->stage1_thread_parser, _next_batch_start); -} - -#endif // SIMDJSON_THREADS_ENABLED - -} // namespace dom - -simdjson_inline simdjson_result::simdjson_result() noexcept - : simdjson_result_base() { -} -simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept - : simdjson_result_base(error) { -} -simdjson_inline simdjson_result::simdjson_result(dom::document_stream &&value) noexcept - : simdjson_result_base(std::forward(value)) { -} - -#if SIMDJSON_EXCEPTIONS -simdjson_inline dom::document_stream::iterator simdjson_result::begin() noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.begin(); -} -simdjson_inline dom::document_stream::iterator simdjson_result::end() noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.end(); -} -#else // SIMDJSON_EXCEPTIONS -#ifndef SIMDJSON_DISABLE_DEPRECATED_API -simdjson_inline dom::document_stream::iterator simdjson_result::begin() noexcept { - first.error = error(); - return first.begin(); -} -simdjson_inline dom::document_stream::iterator simdjson_result::end() noexcept { - first.error = error(); - return first.end(); -} -#endif // SIMDJSON_DISABLE_DEPRECATED_API -#endif // SIMDJSON_EXCEPTIONS - -} // namespace simdjson -#endif // SIMDJSON_DOCUMENT_STREAM_INL_H -/* end file simdjson/dom/document_stream-inl.h */ -/* including simdjson/dom/document-inl.h: #include "simdjson/dom/document-inl.h" */ -/* begin file simdjson/dom/document-inl.h */ -#ifndef SIMDJSON_DOCUMENT_INL_H -#define SIMDJSON_DOCUMENT_INL_H - -// Inline implementations go in here. - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/document.h" */ -/* skipped duplicate #include "simdjson/dom/element-inl.h" */ -/* skipped duplicate #include "simdjson/internal/tape_ref-inl.h" */ -/* including simdjson/internal/jsonformatutils.h: #include "simdjson/internal/jsonformatutils.h" */ -/* begin file simdjson/internal/jsonformatutils.h */ -#ifndef SIMDJSON_INTERNAL_JSONFORMATUTILS_H -#define SIMDJSON_INTERNAL_JSONFORMATUTILS_H - -/* skipped duplicate #include "simdjson/base.h" */ -#include -#include -#include - -namespace simdjson { -namespace internal { - -inline std::ostream& operator<<(std::ostream& out, const escape_json_string &str); - -class escape_json_string { -public: - escape_json_string(std::string_view _str) noexcept : str{_str} {} - operator std::string() const noexcept { std::stringstream s; s << *this; return s.str(); } -private: - std::string_view str; - friend std::ostream& operator<<(std::ostream& out, const escape_json_string &unescaped); -}; - -inline std::ostream& operator<<(std::ostream& out, const escape_json_string &unescaped) { - for (size_t i=0; i(unescaped.str[i]) <= 0x1F) { - // TODO can this be done once at the beginning, or will it mess up << char? - std::ios::fmtflags f(out.flags()); - out << "\\u" << std::hex << std::setw(4) << std::setfill('0') << int(unescaped.str[i]); - out.flags(f); - } else { - out << unescaped.str[i]; - } - } - } - return out; -} - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_JSONFORMATUTILS_H -/* end file simdjson/internal/jsonformatutils.h */ - -#include - -namespace simdjson { -namespace dom { - -// -// document inline implementation -// -inline element document::root() const noexcept { - return element(internal::tape_ref(this, 1)); -} -simdjson_warn_unused -inline size_t document::capacity() const noexcept { - return allocated_capacity; -} - -simdjson_warn_unused -inline error_code document::allocate(size_t capacity) noexcept { - if (capacity == 0) { - string_buf.reset(); - tape.reset(); - allocated_capacity = 0; - return SUCCESS; - } - - // a pathological input like "[[[[..." would generate capacity tape elements, so - // need a capacity of at least capacity + 1, but it is also possible to do - // worse with "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6" - //where capacity + 1 tape elements are - // generated, see issue https://github.com/simdjson/simdjson/issues/345 - size_t tape_capacity = SIMDJSON_ROUNDUP_N(capacity + 3, 64); - // a document with only zero-length strings... could have capacity/3 string - // and we would need capacity/3 * 5 bytes on the string buffer - size_t string_capacity = SIMDJSON_ROUNDUP_N(5 * capacity / 3 + SIMDJSON_PADDING, 64); - string_buf.reset( new (std::nothrow) uint8_t[string_capacity]); - tape.reset(new (std::nothrow) uint64_t[tape_capacity]); - if(!(string_buf && tape)) { - allocated_capacity = 0; - string_buf.reset(); - tape.reset(); - return MEMALLOC; - } - // Technically the allocated_capacity might be larger than capacity - // so the next line is pessimistic. - allocated_capacity = capacity; - return SUCCESS; -} - -inline bool document::dump_raw_tape(std::ostream &os) const noexcept { - uint32_t string_length; - size_t tape_idx = 0; - uint64_t tape_val = tape[tape_idx]; - uint8_t type = uint8_t(tape_val >> 56); - os << tape_idx << " : " << type; - tape_idx++; - size_t how_many = 0; - if (type == 'r') { - how_many = size_t(tape_val & internal::JSON_VALUE_MASK); - } else { - // Error: no starting root node? - return false; - } - os << "\t// pointing to " << how_many << " (right after last node)\n"; - uint64_t payload; - for (; tape_idx < how_many; tape_idx++) { - os << tape_idx << " : "; - tape_val = tape[tape_idx]; - payload = tape_val & internal::JSON_VALUE_MASK; - type = uint8_t(tape_val >> 56); - switch (type) { - case '"': // we have a string - os << "string \""; - std::memcpy(&string_length, string_buf.get() + payload, sizeof(uint32_t)); - os << internal::escape_json_string(std::string_view( - reinterpret_cast(string_buf.get() + payload + sizeof(uint32_t)), - string_length - )); - os << '"'; - os << '\n'; - break; - case 'l': // we have a long int - if (tape_idx + 1 >= how_many) { - return false; - } - os << "integer " << static_cast(tape[++tape_idx]) << "\n"; - break; - case 'u': // we have a long uint - if (tape_idx + 1 >= how_many) { - return false; - } - os << "unsigned integer " << tape[++tape_idx] << "\n"; - break; - case 'd': // we have a double - os << "float "; - if (tape_idx + 1 >= how_many) { - return false; - } - double answer; - std::memcpy(&answer, &tape[++tape_idx], sizeof(answer)); - os << answer << '\n'; - break; - case 'n': // we have a null - os << "null\n"; - break; - case 't': // we have a true - os << "true\n"; - break; - case 'f': // we have a false - os << "false\n"; - break; - case '{': // we have an object - os << "{\t// pointing to next tape location " << uint32_t(payload) - << " (first node after the scope), " - << " saturated count " - << ((payload >> 32) & internal::JSON_COUNT_MASK)<< "\n"; - break; case '}': // we end an object - os << "}\t// pointing to previous tape location " << uint32_t(payload) - << " (start of the scope)\n"; - break; - case '[': // we start an array - os << "[\t// pointing to next tape location " << uint32_t(payload) - << " (first node after the scope), " - << " saturated count " - << ((payload >> 32) & internal::JSON_COUNT_MASK)<< "\n"; - break; - case ']': // we end an array - os << "]\t// pointing to previous tape location " << uint32_t(payload) - << " (start of the scope)\n"; - break; - case 'r': // we start and end with the root node - // should we be hitting the root node? - return false; - case 'Z': // we have a big integer - os << "bigint "; - std::memcpy(&string_length, string_buf.get() + payload, sizeof(uint32_t)); - os << std::string_view( - reinterpret_cast(string_buf.get() + payload + sizeof(uint32_t)), - string_length - ); - os << '\n'; - break; - default: - return false; - } - } - tape_val = tape[tape_idx]; - payload = tape_val & internal::JSON_VALUE_MASK; - type = uint8_t(tape_val >> 56); - os << tape_idx << " : " << type << "\t// pointing to " << payload - << " (start root)\n"; - return true; -} - -} // namespace dom -} // namespace simdjson - -#endif // SIMDJSON_DOCUMENT_INL_H -/* end file simdjson/dom/document-inl.h */ -/* skipped duplicate #include "simdjson/dom/element-inl.h" */ -/* skipped duplicate #include "simdjson/dom/object-inl.h" */ -/* skipped duplicate #include "simdjson/dom/parser-inl.h" */ -/* skipped duplicate #include "simdjson/internal/tape_ref-inl.h" */ -/* including simdjson/dom/serialization-inl.h: #include "simdjson/dom/serialization-inl.h" */ -/* begin file simdjson/dom/serialization-inl.h */ - -#ifndef SIMDJSON_SERIALIZATION_INL_H -#define SIMDJSON_SERIALIZATION_INL_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/parser.h" */ -/* skipped duplicate #include "simdjson/dom/serialization.h" */ -/* skipped duplicate #include "simdjson/internal/tape_type.h" */ - -/* skipped duplicate #include "simdjson/dom/array-inl.h" */ -/* skipped duplicate #include "simdjson/dom/object-inl.h" */ -/* skipped duplicate #include "simdjson/internal/tape_ref-inl.h" */ - -#include - -namespace simdjson { -namespace dom { -inline bool parser::print_json(std::ostream &os) const noexcept { - if (!valid) { - return false; - } - simdjson::internal::string_builder<> sb; - sb.append(doc.root()); - std::string_view answer = sb.str(); - os << answer; - return true; -} - -inline std::ostream &operator<<(std::ostream &out, - simdjson::dom::element value) { - simdjson::internal::string_builder<> sb; - sb.append(value); - return (out << sb.str()); -} -#if SIMDJSON_EXCEPTIONS -inline std::ostream & -operator<<(std::ostream &out, - simdjson::simdjson_result x) { - if (x.error()) { - throw simdjson::simdjson_error(x.error()); - } - return (out << x.value()); -} -#endif -inline std::ostream &operator<<(std::ostream &out, simdjson::dom::array value) { - simdjson::internal::string_builder<> sb; - sb.append(value); - return (out << sb.str()); -} -#if SIMDJSON_EXCEPTIONS -inline std::ostream & -operator<<(std::ostream &out, - simdjson::simdjson_result x) { - if (x.error()) { - throw simdjson::simdjson_error(x.error()); - } - return (out << x.value()); -} -#endif -inline std::ostream &operator<<(std::ostream &out, - simdjson::dom::object value) { - simdjson::internal::string_builder<> sb; - sb.append(value); - return (out << sb.str()); -} -#if SIMDJSON_EXCEPTIONS -inline std::ostream & -operator<<(std::ostream &out, - simdjson::simdjson_result x) { - if (x.error()) { - throw simdjson::simdjson_error(x.error()); - } - return (out << x.value()); -} -#endif - -} // namespace dom - -/*** - * Number utility functions - **/ -namespace { -/**@private - * Escape sequence like \b or \u0001 - * We expect that most compilers will use 8 bytes for this data structure. - **/ -struct escape_sequence { - uint8_t length; - const char - string[7]; // technically, we only ever need 6 characters, we pad to 8 -}; -/**@private - * This converts a signed integer into a character sequence. - * The caller is responsible for providing enough memory (at least - * 20 characters.) - * Though various runtime libraries provide itoa functions, - * it is not part of the C++ standard. The C++17 standard - * adds the to_chars functions which would do as well, but - * we want to support C++11. - */ -static char *fast_itoa(char *output, int64_t value) noexcept { - // This is a standard implementation of itoa. - char buffer[20]; - uint64_t value_positive; - // In general, negating a signed integer is unsafe. - if (value < 0) { - *output++ = '-'; - // Doing value_positive = -value; while avoiding - // undefined behavior warnings. - // It assumes two complement's which is universal at this - // point in time. - std::memcpy(&value_positive, &value, sizeof(value)); - value_positive = (~value_positive) + 1; // this is a negation - } else { - value_positive = value; - } - // We work solely with value_positive. It *might* be easier - // for an optimizing compiler to deal with an unsigned variable - // as far as performance goes. - const char *const end_buffer = buffer + 20; - char *write_pointer = buffer + 19; - // A faster approach is possible if we expect large integers: - // unroll the loop (work in 100s, 1000s) and use some kind of - // memoization. - while (value_positive >= 10) { - *write_pointer-- = char('0' + (value_positive % 10)); - value_positive /= 10; - } - *write_pointer = char('0' + value_positive); - size_t len = end_buffer - write_pointer; - std::memcpy(output, write_pointer, len); - return output + len; -} -/**@private - * This converts an unsigned integer into a character sequence. - * The caller is responsible for providing enough memory (at least - * 19 characters.) - * Though various runtime libraries provide itoa functions, - * it is not part of the C++ standard. The C++17 standard - * adds the to_chars functions which would do as well, but - * we want to support C++11. - */ -static char *fast_itoa(char *output, uint64_t value) noexcept { - // This is a standard implementation of itoa. - char buffer[20]; - const char *const end_buffer = buffer + 20; - char *write_pointer = buffer + 19; - // A faster approach is possible if we expect large integers: - // unroll the loop (work in 100s, 1000s) and use some kind of - // memoization. - while (value >= 10) { - *write_pointer-- = char('0' + (value % 10)); - value /= 10; - }; - *write_pointer = char('0' + value); - size_t len = end_buffer - write_pointer; - std::memcpy(output, write_pointer, len); - return output + len; -} - -} // anonymous namespace -namespace internal { - -/*** - * Minifier/formatter code. - **/ - -template -simdjson_inline void base_formatter::number(uint64_t x) { - char number_buffer[24]; - char *newp = fast_itoa(number_buffer, x); - chars(number_buffer, newp); -} - -template -simdjson_inline void base_formatter::number(int64_t x) { - char number_buffer[24]; - char *newp = fast_itoa(number_buffer, x); - chars(number_buffer, newp); -} - -template -simdjson_inline void base_formatter::number(double x) { - char number_buffer[24]; - // Currently, passing the nullptr to the second argument is - // safe because our implementation does not check the second - // argument. - char *newp = internal::to_chars(number_buffer, nullptr, x); - chars(number_buffer, newp); -} - -template -simdjson_inline void base_formatter::start_array() { - one_char('['); -} - -template -simdjson_inline void base_formatter::end_array() { - one_char(']'); -} - -template -simdjson_inline void base_formatter::start_object() { - one_char('{'); -} - -template -simdjson_inline void base_formatter::end_object() { - one_char('}'); -} - -template -simdjson_inline void base_formatter::comma() { - one_char(','); -} - -template -simdjson_inline void base_formatter::true_atom() { - const char *s = "true"; - chars(s, s + 4); -} - -template -simdjson_inline void base_formatter::false_atom() { - const char *s = "false"; - chars(s, s + 5); -} - -template -simdjson_inline void base_formatter::null_atom() { - const char *s = "null"; - chars(s, s + 4); -} - -template -simdjson_inline void base_formatter::one_char(char c) { - buffer.push_back(c); -} - -template -simdjson_inline void base_formatter::chars(const char *begin, - const char *end) { - buffer.append(begin, end); -} - -template -simdjson_inline void -base_formatter::key(std::string_view unescaped) { - string(unescaped); - one_char(':'); -} - -template -simdjson_inline void -base_formatter::string(std::string_view unescaped) { - one_char('\"'); - size_t i = 0; - // Fast path for the case where we have no control character, no ", and no - // backslash. This should include most keys. - // - // We would like to use 'bool' but some compilers take offense to bitwise - // operation with bool types. - constexpr static char needs_escaping[] = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - for (; i + 8 <= unescaped.length(); i += 8) { - // Poor's man vectorization. This could get much faster if we used SIMD. - // - // It is not the case that replacing '|' with '||' would be neutral - // performance-wise. - if (needs_escaping[uint8_t(unescaped[i])] | - needs_escaping[uint8_t(unescaped[i + 1])] | - needs_escaping[uint8_t(unescaped[i + 2])] | - needs_escaping[uint8_t(unescaped[i + 3])] | - needs_escaping[uint8_t(unescaped[i + 4])] | - needs_escaping[uint8_t(unescaped[i + 5])] | - needs_escaping[uint8_t(unescaped[i + 6])] | - needs_escaping[uint8_t(unescaped[i + 7])]) { - break; - } - } - for (; i < unescaped.length(); i++) { - if (needs_escaping[uint8_t(unescaped[i])]) { - break; - } - } - // The following is also possible and omits a 256-byte table, but it is - // slower: for (; (i < unescaped.length()) && (uint8_t(unescaped[i]) > 0x1F) - // && (unescaped[i] != '\"') && (unescaped[i] != '\\'); i++) {} - - // At least for long strings, the following should be fast. We could - // do better by integrating the checks and the insertion. - chars(unescaped.data(), unescaped.data() + i); - // We caught a control character if we enter this loop (slow). - // Note that we are do not restart from the beginning, but rather we continue - // from the point where we encountered something that requires escaping. - for (; i < unescaped.length(); i++) { - switch (unescaped[i]) { - case '\"': { - const char *s = "\\\""; - chars(s, s + 2); - } break; - case '\\': { - const char *s = "\\\\"; - chars(s, s + 2); - } break; - default: - if (uint8_t(unescaped[i]) <= 0x1F) { - // If packed, this uses 8 * 32 bytes. - // Note that we expect most compilers to embed this code in the data - // section. - constexpr static escape_sequence escaped[32] = { - {6, "\\u0000"}, {6, "\\u0001"}, {6, "\\u0002"}, {6, "\\u0003"}, - {6, "\\u0004"}, {6, "\\u0005"}, {6, "\\u0006"}, {6, "\\u0007"}, - {2, "\\b"}, {2, "\\t"}, {2, "\\n"}, {6, "\\u000b"}, - {2, "\\f"}, {2, "\\r"}, {6, "\\u000e"}, {6, "\\u000f"}, - {6, "\\u0010"}, {6, "\\u0011"}, {6, "\\u0012"}, {6, "\\u0013"}, - {6, "\\u0014"}, {6, "\\u0015"}, {6, "\\u0016"}, {6, "\\u0017"}, - {6, "\\u0018"}, {6, "\\u0019"}, {6, "\\u001a"}, {6, "\\u001b"}, - {6, "\\u001c"}, {6, "\\u001d"}, {6, "\\u001e"}, {6, "\\u001f"}}; - auto u = escaped[uint8_t(unescaped[i])]; - chars(u.string, u.string + u.length); - } else { - one_char(unescaped[i]); - } - } // switch - } // for - one_char('\"'); -} - -template inline void base_formatter::clear() { - buffer.clear(); -} - -template -simdjson_inline std::string_view base_formatter::str() const { - return buffer.str(); -} - -simdjson_inline void mini_formatter::print_newline() { return; } - -simdjson_inline void mini_formatter::print_indents(size_t depth) { - (void)depth; - return; -} - -simdjson_inline void mini_formatter::print_space() { return; } - -simdjson_inline void pretty_formatter::print_newline() { one_char('\n'); } - -simdjson_inline void pretty_formatter::print_indents(size_t depth) { - if (this->indent_step <= 0) { - return; - } - for (size_t i = 0; i < this->indent_step * depth; i++) { - one_char(' '); - } -} - -simdjson_inline void pretty_formatter::print_space() { one_char(' '); } - -/*** - * String building code. - **/ - -template -inline void string_builder::append(simdjson::dom::element value) { - // using tape_type = simdjson::internal::tape_type; - size_t depth = 0; - constexpr size_t MAX_DEPTH = 16; - bool is_object[MAX_DEPTH]; - is_object[0] = false; - bool after_value = false; - - internal::tape_ref iter(value.tape); - do { - // print commas after each value - if (after_value) { - format.comma(); - format.print_newline(); - } - - format.print_indents(depth); - - // If we are in an object, print the next key and :, and skip to the next - // value. - if (is_object[depth]) { - format.key(iter.get_string_view()); - format.print_space(); - iter.json_index++; - } - switch (iter.tape_ref_type()) { - - // Arrays - case tape_type::START_ARRAY: { - // If we're too deep, we need to recurse to go deeper. - depth++; - if (simdjson_unlikely(depth >= MAX_DEPTH)) { - append(simdjson::dom::array(iter)); - iter.json_index = iter.matching_brace_index() - 1; // Jump to the ] - depth--; - break; - } - - // Output start [ - format.start_array(); - iter.json_index++; - - // Handle empty [] (we don't want to come back around and print commas) - if (iter.tape_ref_type() == tape_type::END_ARRAY) { - format.end_array(); - depth--; - break; - } - - is_object[depth] = false; - after_value = false; - format.print_newline(); - continue; - } - - // Objects - case tape_type::START_OBJECT: { - // If we're too deep, we need to recurse to go deeper. - depth++; - if (simdjson_unlikely(depth >= MAX_DEPTH)) { - append(simdjson::dom::object(iter)); - iter.json_index = iter.matching_brace_index() - 1; // Jump to the } - depth--; - break; - } - - // Output start { - format.start_object(); - iter.json_index++; - - // Handle empty {} (we don't want to come back around and print commas) - if (iter.tape_ref_type() == tape_type::END_OBJECT) { - format.end_object(); - depth--; - break; - } - - is_object[depth] = true; - after_value = false; - format.print_newline(); - continue; - } - - // Scalars - case tape_type::STRING: - format.string(iter.get_string_view()); - break; - case tape_type::BIGINT: { - // Big integer stored as string — output raw digits (no quotes) - auto sv = iter.get_string_view(); - format.chars(sv.data(), sv.data() + sv.size()); - break; - } - case tape_type::INT64: - format.number(iter.next_tape_value()); - iter.json_index++; // numbers take up 2 spots, so we need to increment - // extra - break; - case tape_type::UINT64: - format.number(iter.next_tape_value()); - iter.json_index++; // numbers take up 2 spots, so we need to increment - // extra - break; - case tape_type::DOUBLE: - format.number(iter.next_tape_value()); - iter.json_index++; // numbers take up 2 spots, so we need to increment - // extra - break; - case tape_type::TRUE_VALUE: - format.true_atom(); - break; - case tape_type::FALSE_VALUE: - format.false_atom(); - break; - case tape_type::NULL_VALUE: - format.null_atom(); - break; - - // These are impossible - case tape_type::END_ARRAY: - case tape_type::END_OBJECT: - case tape_type::ROOT: - SIMDJSON_UNREACHABLE(); - } - iter.json_index++; - after_value = true; - - // Handle multiple ends in a row - while (depth != 0 && (iter.tape_ref_type() == tape_type::END_ARRAY || - iter.tape_ref_type() == tape_type::END_OBJECT)) { - format.print_newline(); - depth--; - format.print_indents(depth); - if (iter.tape_ref_type() == tape_type::END_ARRAY) { - format.end_array(); - } else { - format.end_object(); - } - iter.json_index++; - } - - // Stop when we're at depth 0 - } while (depth != 0); - - format.print_newline(); -} - -template -inline void string_builder::append(simdjson::dom::object value) { - format.start_object(); - auto pair = value.begin(); - auto end = value.end(); - if (pair != end) { - append(*pair); - for (++pair; pair != end; ++pair) { - format.comma(); - append(*pair); - } - } - format.end_object(); -} - -template -inline void string_builder::append(simdjson::dom::array value) { - format.start_array(); - auto iter = value.begin(); - auto end = value.end(); - if (iter != end) { - append(*iter); - for (++iter; iter != end; ++iter) { - format.comma(); - append(*iter); - } - } - format.end_array(); -} - -template -simdjson_inline void -string_builder::append(simdjson::dom::key_value_pair kv) { - format.key(kv.key); - append(kv.value); -} - -template -simdjson_inline void string_builder::clear() { - format.clear(); -} - -template -simdjson_inline std::string_view string_builder::str() const { - return format.str(); -} - -} // namespace internal -} // namespace simdjson - -#endif -/* end file simdjson/dom/serialization-inl.h */ -/* including simdjson/dom/fractured_json-inl.h: #include "simdjson/dom/fractured_json-inl.h" */ -/* begin file simdjson/dom/fractured_json-inl.h */ -#ifndef SIMDJSON_DOM_FRACTURED_JSON_INL_H -#define SIMDJSON_DOM_FRACTURED_JSON_INL_H - -/* skipped duplicate #include "simdjson/dom/fractured_json.h" */ -/* skipped duplicate #include "simdjson/dom/serialization.h" */ -/* skipped duplicate #include "simdjson/dom/element-inl.h" */ -/* skipped duplicate #include "simdjson/dom/array-inl.h" */ -/* skipped duplicate #include "simdjson/dom/object-inl.h" */ -/* skipped duplicate #include "simdjson/dom/parser-inl.h" */ -/* skipped duplicate #include "simdjson/padded_string.h" */ -/* including simdjson/internal/json_structure_analyzer.h: #include "simdjson/internal/json_structure_analyzer.h" */ -/* begin file simdjson/internal/json_structure_analyzer.h */ -#ifndef SIMDJSON_INTERNAL_JSON_STRUCTURE_ANALYZER_H -#define SIMDJSON_INTERNAL_JSON_STRUCTURE_ANALYZER_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/element.h" */ -/* skipped duplicate #include "simdjson/dom/array.h" */ -/* skipped duplicate #include "simdjson/dom/object.h" */ -/* skipped duplicate #include "simdjson/dom/fractured_json.h" */ -/* skipped duplicate #include "simdjson/internal/tape_type.h" */ - -#include -#include -#include -#include - -namespace simdjson { -namespace internal { - -/** - * Layout mode for fractured JSON formatting. - */ -enum class layout_mode { - INLINE, // Single line: [1, 2, 3] or {"a": 1} - COMPACT_MULTILINE, // Multiple items per line with breaks - TABLE, // Tabular format for arrays of similar objects - EXPANDED // Traditional multi-line with indentation -}; - -/** - * Metrics computed for a JSON element during structure analysis. - * These metrics drive layout decisions and contain child metrics for recursive formatting. - */ -struct element_metrics { - /** Nesting depth score (0 = scalar, 1 = flat container, etc.) */ - size_t complexity = 0; - - /** Estimated character length if rendered inline (minified + spaces) */ - size_t estimated_inline_len = 0; - - /** Number of direct children (0 for scalars) */ - size_t child_count = 0; - - /** Pre-computed: can this element be rendered inline? */ - bool can_inline = false; - - /** Is this an array where all elements have similar structure? */ - bool is_uniform_array = false; - - /** For uniform arrays of objects: the common keys */ - std::vector common_keys{}; - - /** Recommended layout mode based on analysis */ - layout_mode recommended_layout = layout_mode::EXPANDED; - - /** Child metrics for arrays and objects (in order of iteration) */ - std::vector children{}; -}; - -/** - * Analyzes JSON structure to compute metrics for formatting decisions. - * - * The analyzer performs a single pass over the DOM to compute: - * - Complexity (nesting depth) - * - Estimated inline length - * - Array uniformity for table detection - * - * Metrics are stored hierarchically with child metrics embedded in parent metrics, - * enabling efficient lookup during formatting without address-based caching. - */ -class structure_analyzer { -public: - /** Default constructor */ - structure_analyzer() : current_opts_(nullptr) {} - - /** Copy constructor - deleted since class has pointer member */ - structure_analyzer(const structure_analyzer&) = delete; - - /** Copy assignment - deleted since class has pointer member */ - structure_analyzer& operator=(const structure_analyzer&) = delete; - - /** Move constructor */ - structure_analyzer(structure_analyzer&&) = default; - - /** Move assignment */ - structure_analyzer& operator=(structure_analyzer&&) = default; - - /** - * Analyze a DOM element and compute metrics. - * @param elem The element to analyze - * @param opts Formatting options that affect metric computation - * @return Metrics for the root element (with child metrics embedded) - */ - element_metrics analyze(const dom::element& elem, - const fractured_json_options& opts); - - /** - * Clear state. - */ - void clear(); - - /** - * Analyze an array element directly (for standalone array formatting). - * @param arr The array to analyze - * @param opts Formatting options - * @return Metrics for the array - */ - element_metrics analyze_array(const dom::array& arr, - const fractured_json_options& opts); - - /** - * Analyze an object element directly (for standalone object formatting). - * @param obj The object to analyze - * @param opts Formatting options - * @return Metrics for the object - */ - element_metrics analyze_object(const dom::object& obj, - const fractured_json_options& opts); - -private: - const fractured_json_options* current_opts_ = nullptr; - - /** Recursive analysis implementation */ - element_metrics analyze_element(const dom::element& elem, size_t depth); - - /** Analyze scalar values (strings, numbers, booleans, null) */ - element_metrics analyze_scalar(const dom::element& elem); - - /** Analyze an array element */ - element_metrics analyze_array(const dom::array& arr, size_t depth); - - /** Analyze an object element */ - element_metrics analyze_object(const dom::object& obj, size_t depth); - - /** Estimate inline length for a string (including quotes and escaping) */ - size_t estimate_string_length(std::string_view s) const; - - /** Estimate inline length for a number */ - size_t estimate_number_length(double d) const; - size_t estimate_number_length(int64_t i) const; - size_t estimate_number_length(uint64_t u) const; - - /** - * Check if an array contains uniform objects suitable for table formatting. - * @param arr The array to check - * @param common_keys Output: keys common to all objects - * @return true if the array is suitable for table formatting - */ - bool check_array_uniformity(const dom::array& arr, - std::vector& common_keys) const; - - /** - * Compute similarity between two objects. - * @return Fraction of keys that are common (0.0 to 1.0) - */ - double compute_object_similarity(const dom::object& a, - const dom::object& b) const; - - /** - * Decide the recommended layout mode based on metrics and options. - */ - layout_mode decide_layout(const element_metrics& metrics, - size_t depth, - size_t available_width) const; -}; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_JSON_STRUCTURE_ANALYZER_H -/* end file simdjson/internal/json_structure_analyzer.h */ -/* including simdjson/internal/fractured_formatter.h: #include "simdjson/internal/fractured_formatter.h" */ -/* begin file simdjson/internal/fractured_formatter.h */ -#ifndef SIMDJSON_INTERNAL_FRACTURED_FORMATTER_H -#define SIMDJSON_INTERNAL_FRACTURED_FORMATTER_H - -/* skipped duplicate #include "simdjson/dom/serialization.h" */ -/* skipped duplicate #include "simdjson/dom/fractured_json.h" */ -/* skipped duplicate #include "simdjson/internal/json_structure_analyzer.h" */ - -namespace simdjson { -namespace internal { - -/** - * Fractured JSON formatter using CRTP pattern. - * - * This formatter intelligently chooses between different layout modes - * (inline, compact multiline, table, expanded) based on pre-computed - * structure metrics. - */ -class fractured_formatter : public base_formatter { -public: - explicit fractured_formatter(const fractured_json_options& opts = {}); - - /** CRTP hook: print newline (context-aware) */ - simdjson_inline void print_newline(); - - /** CRTP hook: print indentation */ - simdjson_inline void print_indents(size_t depth); - - /** CRTP hook: print space (context-aware) */ - simdjson_inline void print_space(); - - /** Set the current layout mode */ - void set_layout_mode(layout_mode mode); - - /** Get the current layout mode */ - layout_mode get_layout_mode() const; - - /** Set current depth for formatting decisions */ - void set_depth(size_t depth); - - /** Get current depth */ - size_t get_depth() const; - - /** Track current line length for compact multiline decisions */ - void track_line_length(size_t chars); - - /** Reset line length (after newline) */ - void reset_line_length(); - - /** Get current line length */ - size_t get_line_length() const; - - /** Check if we should break to a new line in compact mode */ - bool should_break_line(size_t upcoming_length) const; - - /** Get the options */ - const fractured_json_options& options() const; - - // Table formatting support - /** Begin a table row */ - void begin_table_row(); - - /** End a table row */ - void end_table_row(); - - /** Set column widths for table alignment */ - void set_column_widths(const std::vector& widths); - - /** Get current column index in table mode */ - size_t get_column_index() const; - - /** Advance to next column */ - void next_column(); - - /** Add padding to align with column width */ - void align_to_column_width(size_t actual_width); - -private: - fractured_json_options options_; - layout_mode current_layout_ = layout_mode::EXPANDED; - size_t current_depth_ = 0; - size_t current_line_length_ = 0; - - // Table state - bool in_table_mode_ = false; - std::vector column_widths_; - size_t current_column_ = 0; -}; - -/** - * Specialized string builder for fractured JSON formatting. - * - * This builder performs two passes: - * 1. Analyze the structure to compute metrics - * 2. Format using the metrics to make layout decisions - */ -class fractured_string_builder { -public: - fractured_string_builder(const fractured_json_options& opts = {}); - - /** Append a DOM element with fractured formatting */ - void append(const dom::element& value); - - /** Append a DOM array with fractured formatting */ - void append(const dom::array& value); - - /** Append a DOM object with fractured formatting */ - void append(const dom::object& value); - - /** Clear the builder */ - simdjson_inline void clear(); - - /** Get the formatted string */ - simdjson_inline std::string_view str() const; - -private: - fractured_formatter format_; - structure_analyzer analyzer_; - fractured_json_options options_; - - /** Format an element using pre-computed metrics */ - void format_element(const dom::element& elem, const element_metrics& metrics, size_t depth); - - /** Format an array with the appropriate layout */ - void format_array(const dom::array& arr, const element_metrics& metrics, size_t depth); - - /** Format an array inline: [1, 2, 3] */ - void format_array_inline(const dom::array& arr, const element_metrics& metrics); - - /** Format an array with compact multiline: multiple items per line */ - void format_array_compact_multiline(const dom::array& arr, const element_metrics& metrics, size_t depth); - - /** Format an array as a table */ - void format_array_as_table(const dom::array& arr, const element_metrics& metrics, size_t depth); - - /** Format an array expanded: one item per line */ - void format_array_expanded(const dom::array& arr, const element_metrics& metrics, size_t depth); - - /** Format an object with the appropriate layout */ - void format_object(const dom::object& obj, const element_metrics& metrics, size_t depth); - - /** Format an object inline: {"a": 1, "b": 2} */ - void format_object_inline(const dom::object& obj, const element_metrics& metrics); - - /** Format an object expanded: one key per line */ - void format_object_expanded(const dom::object& obj, const element_metrics& metrics, size_t depth); - - /** Format a scalar value */ - void format_scalar(const dom::element& elem); - - /** Calculate column widths for table formatting */ - std::vector calculate_column_widths(const dom::array& arr, - const std::vector& columns) const; - - /** Measure the actual formatted length of a value (for alignment) */ - size_t measure_value_length(const dom::element& elem) const; -}; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_FRACTURED_FORMATTER_H -/* end file simdjson/internal/fractured_formatter.h */ - -#include -#include -#include - -namespace simdjson { -namespace internal { - -// -// Structure Analyzer Implementation -// - -inline element_metrics structure_analyzer::analyze(const dom::element& elem, - const fractured_json_options& opts) { - current_opts_ = &opts; - return analyze_element(elem, 0); -} - -inline void structure_analyzer::clear() { - current_opts_ = nullptr; -} - -inline element_metrics structure_analyzer::analyze_array(const dom::array& arr, - const fractured_json_options& opts) { - current_opts_ = &opts; - return analyze_array(arr, 0); -} - -inline element_metrics structure_analyzer::analyze_object(const dom::object& obj, - const fractured_json_options& opts) { - current_opts_ = &opts; - return analyze_object(obj, 0); -} - -inline element_metrics structure_analyzer::analyze_element(const dom::element& elem, size_t depth) { - switch (elem.type()) { - case dom::element_type::ARRAY: { - dom::array arr; - if (elem.get_array().get(arr) == SUCCESS) { - return analyze_array(arr, depth); - } - break; - } - case dom::element_type::OBJECT: { - dom::object obj; - if (elem.get_object().get(obj) == SUCCESS) { - return analyze_object(obj, depth); - } - break; - } - default: - // Handle all scalar types with a helper - return analyze_scalar(elem); - } - return element_metrics{}; -} - -inline element_metrics structure_analyzer::analyze_scalar(const dom::element& elem) { - element_metrics metrics; - metrics.complexity = 0; - metrics.child_count = 0; - metrics.can_inline = true; - metrics.recommended_layout = layout_mode::INLINE; - - switch (elem.type()) { - case dom::element_type::STRING: { - std::string_view str; - if (elem.get_string().get(str) == SUCCESS) { - metrics.estimated_inline_len = estimate_string_length(str); - } - break; - } - case dom::element_type::INT64: { - int64_t val; - if (elem.get_int64().get(val) == SUCCESS) { - metrics.estimated_inline_len = estimate_number_length(val); - } - break; - } - case dom::element_type::UINT64: { - uint64_t val; - if (elem.get_uint64().get(val) == SUCCESS) { - metrics.estimated_inline_len = estimate_number_length(val); - } - break; - } - case dom::element_type::DOUBLE: { - double val; - if (elem.get_double().get(val) == SUCCESS) { - metrics.estimated_inline_len = estimate_number_length(val); - } - break; - } - case dom::element_type::BOOL: { - bool val; - if (elem.get_bool().get(val) == SUCCESS) { - metrics.estimated_inline_len = val ? 4 : 5; // "true" or "false" - } - break; - } - case dom::element_type::NULL_VALUE: - metrics.estimated_inline_len = 4; // "null" - break; - default: - break; - } - - return metrics; -} - -inline element_metrics structure_analyzer::analyze_array(const dom::array& arr, - size_t depth) { - element_metrics metrics; - metrics.complexity = 1; // At least 1 for being an array - metrics.estimated_inline_len = 2; // "[]" - metrics.child_count = 0; - - size_t max_child_complexity = 0; - bool first = true; - - for (dom::element child : arr) { - if (!first) { - metrics.estimated_inline_len += 2; // ", " - } - first = false; - - element_metrics child_metrics = analyze_element(child, depth + 1); - metrics.estimated_inline_len += child_metrics.estimated_inline_len; - max_child_complexity = (std::max)(max_child_complexity, child_metrics.complexity); - metrics.child_count++; - metrics.children.push_back(std::move(child_metrics)); - } - - // Complexity is 1 + max child complexity - metrics.complexity = 1 + max_child_complexity; - - // Check if can inline - metrics.can_inline = (metrics.complexity <= current_opts_->max_inline_complexity) && - (metrics.estimated_inline_len <= current_opts_->max_inline_length); - - // Check for uniform array (table formatting) - if (current_opts_->enable_table_format && - metrics.child_count >= current_opts_->min_table_rows) { - metrics.is_uniform_array = check_array_uniformity(arr, metrics.common_keys); - } - - // Decide layout - if (metrics.child_count == 0) { - metrics.recommended_layout = layout_mode::INLINE; - } else if (metrics.can_inline) { - metrics.recommended_layout = layout_mode::INLINE; - } else if (metrics.is_uniform_array && !metrics.common_keys.empty()) { - metrics.recommended_layout = layout_mode::TABLE; - } else if (current_opts_->enable_compact_multiline && - max_child_complexity <= current_opts_->max_compact_array_complexity) { - metrics.recommended_layout = layout_mode::COMPACT_MULTILINE; - } else { - metrics.recommended_layout = layout_mode::EXPANDED; - } - - return metrics; -} - -inline element_metrics structure_analyzer::analyze_object(const dom::object& obj, - size_t depth) { - element_metrics metrics; - metrics.complexity = 1; - metrics.estimated_inline_len = 2; // "{}" - metrics.child_count = 0; - - size_t max_child_complexity = 0; - bool first = true; - - for (dom::key_value_pair field : obj) { - if (!first) { - metrics.estimated_inline_len += 2; // ", " - } - first = false; - - // Key length: quotes + key + colon + space - metrics.estimated_inline_len += estimate_string_length(field.key) + 2; - - element_metrics child_metrics = analyze_element(field.value, depth + 1); - metrics.estimated_inline_len += child_metrics.estimated_inline_len; - max_child_complexity = (std::max)(max_child_complexity, child_metrics.complexity); - metrics.child_count++; - metrics.children.push_back(std::move(child_metrics)); - } - - metrics.complexity = 1 + max_child_complexity; - - metrics.can_inline = (metrics.complexity <= current_opts_->max_inline_complexity) && - (metrics.estimated_inline_len <= current_opts_->max_inline_length); - - // Objects use inline or expanded (no table/compact for objects) - if (metrics.child_count == 0 || metrics.can_inline) { - metrics.recommended_layout = layout_mode::INLINE; - } else { - metrics.recommended_layout = layout_mode::EXPANDED; - } - - return metrics; -} - -inline size_t structure_analyzer::estimate_string_length(std::string_view s) const { - size_t len = 2; // quotes - for (char c : s) { - if (c == '"' || c == '\\' || static_cast(c) < 32) { - len += 2; // escape sequence (at least) - } else { - len += 1; - } - } - return len; -} - -inline size_t structure_analyzer::estimate_number_length(double d) const { - if (std::isnan(d) || std::isinf(d)) { - return 4; // "null" for invalid numbers - } - // Rough estimate: up to 17 significant digits + sign + decimal point + exponent - char buf[32]; - int len = snprintf(buf, sizeof(buf), "%.17g", d); - return len > 0 ? static_cast(len) : 20; -} - -inline size_t structure_analyzer::estimate_number_length(int64_t i) const { - if (i == 0) return 1; - // Handle INT64_MIN specially to avoid overflow when negating - if (i == INT64_MIN) return 20; // "-9223372036854775808" is 20 characters - size_t len = (i < 0) ? 1 : 0; // negative sign - int64_t abs_val = (i < 0) ? -i : i; - while (abs_val > 0) { - len++; - abs_val /= 10; - } - return len; -} - -inline size_t structure_analyzer::estimate_number_length(uint64_t u) const { - if (u == 0) return 1; - size_t len = 0; - while (u > 0) { - len++; - u /= 10; - } - return len; -} - -inline bool structure_analyzer::check_array_uniformity(const dom::array& arr, - std::vector& common_keys) const { - common_keys.clear(); - - std::set shared_keys; - dom::object first_obj; - bool have_first = false; - size_t object_count = 0; - - for (dom::element elem : arr) { - if (elem.type() != dom::element_type::OBJECT) { - return false; // Not all elements are objects - } - - dom::object obj; - if (elem.get_object().get(obj) != SUCCESS) { - return false; - } - - std::set current_keys; - for (dom::key_value_pair field : obj) { - current_keys.insert(std::string(field.key)); - } - - if (!have_first) { - shared_keys = current_keys; - first_obj = obj; - have_first = true; - } else { - // Check similarity threshold against the first object - double similarity = compute_object_similarity(first_obj, obj); - if (similarity < current_opts_->table_similarity_threshold) { - return false; // Objects are too dissimilar for table format - } - - // Intersect with current keys - std::set intersection; - std::set_intersection(shared_keys.begin(), shared_keys.end(), - current_keys.begin(), current_keys.end(), - std::inserter(intersection, intersection.begin())); - shared_keys = intersection; - } - - object_count++; - } - - if (object_count < current_opts_->min_table_rows) { - return false; - } - - // Require at least one common key for table formatting - if (shared_keys.empty()) { - return false; - } - - common_keys.assign(shared_keys.begin(), shared_keys.end()); - return true; -} - -inline double structure_analyzer::compute_object_similarity(const dom::object& a, - const dom::object& b) const { - std::set keys_a, keys_b; - for (dom::key_value_pair field : a) { - keys_a.insert(std::string(field.key)); - } - for (dom::key_value_pair field : b) { - keys_b.insert(std::string(field.key)); - } - - std::set intersection; - std::set_intersection(keys_a.begin(), keys_a.end(), - keys_b.begin(), keys_b.end(), - std::inserter(intersection, intersection.begin())); - - std::set union_set; - std::set_union(keys_a.begin(), keys_a.end(), - keys_b.begin(), keys_b.end(), - std::inserter(union_set, union_set.begin())); - - if (union_set.empty()) return 1.0; - return static_cast(intersection.size()) / static_cast(union_set.size()); -} - -inline layout_mode structure_analyzer::decide_layout(const element_metrics& metrics, - size_t depth, - size_t available_width) const { - if (metrics.child_count == 0) { - return layout_mode::INLINE; - } - - // Check inline feasibility - size_t indent_width = depth * current_opts_->indent_spaces; - if (metrics.can_inline && - metrics.estimated_inline_len + indent_width <= available_width) { - return layout_mode::INLINE; - } - - // Check table mode - if (metrics.is_uniform_array && !metrics.common_keys.empty()) { - return layout_mode::TABLE; - } - - // Check compact multiline - if (current_opts_->enable_compact_multiline && - metrics.complexity <= current_opts_->max_compact_array_complexity + 1) { - return layout_mode::COMPACT_MULTILINE; - } - - return layout_mode::EXPANDED; -} - -// -// Fractured Formatter Implementation -// - -inline fractured_formatter::fractured_formatter(const fractured_json_options& opts) - : options_(opts), column_widths_{} {} - -simdjson_inline void fractured_formatter::print_newline() { - if (current_layout_ == layout_mode::INLINE) { - return; // No newlines in inline mode - } - one_char('\n'); - current_line_length_ = 0; -} - -simdjson_inline void fractured_formatter::print_indents(size_t depth) { - if (current_layout_ == layout_mode::INLINE) { - return; // No indentation in inline mode - } - for (size_t i = 0; i < depth * options_.indent_spaces; i++) { - one_char(' '); - current_line_length_++; - } -} - -simdjson_inline void fractured_formatter::print_space() { - one_char(' '); - current_line_length_++; -} - -inline void fractured_formatter::set_layout_mode(layout_mode mode) { - current_layout_ = mode; -} - -inline layout_mode fractured_formatter::get_layout_mode() const { - return current_layout_; -} - -inline void fractured_formatter::set_depth(size_t depth) { - current_depth_ = depth; -} - -inline size_t fractured_formatter::get_depth() const { - return current_depth_; -} - -inline void fractured_formatter::track_line_length(size_t chars) { - current_line_length_ += chars; -} - -inline void fractured_formatter::reset_line_length() { - current_line_length_ = 0; -} - -inline size_t fractured_formatter::get_line_length() const { - return current_line_length_; -} - -inline bool fractured_formatter::should_break_line(size_t upcoming_length) const { - return (current_line_length_ + upcoming_length) > options_.max_total_line_length; -} - -inline const fractured_json_options& fractured_formatter::options() const { - return options_; -} - -inline void fractured_formatter::begin_table_row() { - in_table_mode_ = true; - current_column_ = 0; -} - -inline void fractured_formatter::end_table_row() { - in_table_mode_ = false; - current_column_ = 0; -} - -inline void fractured_formatter::set_column_widths(const std::vector& widths) { - column_widths_ = widths; -} - -inline size_t fractured_formatter::get_column_index() const { - return current_column_; -} - -inline void fractured_formatter::next_column() { - current_column_++; -} - -inline void fractured_formatter::align_to_column_width(size_t actual_width) { - if (current_column_ < column_widths_.size()) { - size_t target_width = column_widths_[current_column_]; - while (actual_width < target_width) { - one_char(' '); - actual_width++; - current_line_length_++; - } - } -} - -// -// Fractured String Builder Implementation -// - -inline fractured_string_builder::fractured_string_builder(const fractured_json_options& opts) - : format_(opts), analyzer_{}, options_(opts) {} - -inline void fractured_string_builder::append(const dom::element& value) { - // Phase 1: Analyze structure (metrics tree is built recursively) - element_metrics root_metrics = analyzer_.analyze(value, options_); - - // Phase 2: Format using metrics tree (passed through recursion) - format_element(value, root_metrics, 0); -} - -inline void fractured_string_builder::append(const dom::array& value) { - // Analyze the array to get proper metrics with children - element_metrics metrics = analyzer_.analyze_array(value, options_); - format_array(value, metrics, 0); -} - -inline void fractured_string_builder::append(const dom::object& value) { - // Analyze the object to get proper metrics with children - element_metrics metrics = analyzer_.analyze_object(value, options_); - format_object(value, metrics, 0); -} - -simdjson_inline void fractured_string_builder::clear() { - format_.clear(); - analyzer_.clear(); -} - -simdjson_inline std::string_view fractured_string_builder::str() const { - return format_.str(); -} - -inline void fractured_string_builder::format_element(const dom::element& elem, - const element_metrics& metrics, - size_t depth) { - switch (elem.type()) { - case dom::element_type::ARRAY: { - dom::array arr; - if (elem.get_array().get(arr) == SUCCESS) { - format_array(arr, metrics, depth); - } - break; - } - case dom::element_type::OBJECT: { - dom::object obj; - if (elem.get_object().get(obj) == SUCCESS) { - format_object(obj, metrics, depth); - } - break; - } - default: - format_scalar(elem); - break; - } -} - -inline void fractured_string_builder::format_array(const dom::array& arr, - const element_metrics& metrics, - size_t depth) { - switch (metrics.recommended_layout) { - case layout_mode::INLINE: - format_array_inline(arr, metrics); - break; - case layout_mode::COMPACT_MULTILINE: - format_array_compact_multiline(arr, metrics, depth); - break; - case layout_mode::TABLE: - format_array_as_table(arr, metrics, depth); - break; - case layout_mode::EXPANDED: - default: - format_array_expanded(arr, metrics, depth); - break; - } -} - -inline void fractured_string_builder::format_array_inline(const dom::array& arr, - const element_metrics& metrics) { - layout_mode prev_layout = format_.get_layout_mode(); - format_.set_layout_mode(layout_mode::INLINE); - - format_.start_array(); - - bool first = true; - bool empty = true; - size_t child_idx = 0; - for (dom::element elem : arr) { - empty = false; - if (!first) { - format_.comma(); - if (options_.comma_padding) { - format_.print_space(); - } - } else if (options_.simple_bracket_padding) { - format_.print_space(); - } - first = false; - const element_metrics& child_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; - format_element(elem, child_metrics, 0); - child_idx++; - } - - if (options_.simple_bracket_padding && !empty) { - format_.print_space(); - } - format_.end_array(); - - format_.set_layout_mode(prev_layout); -} - -inline void fractured_string_builder::format_array_compact_multiline(const dom::array& arr, - const element_metrics& metrics, - size_t depth) { - format_.start_array(); - format_.print_newline(); - format_.print_indents(depth + 1); - - size_t items_on_line = 0; - bool first = true; - size_t child_idx = 0; - - for (dom::element elem : arr) { - if (!first) { - format_.comma(); - - // Check if we should break to new line - if (items_on_line >= options_.max_items_per_line || - format_.should_break_line(20)) { // 20 is rough estimate for next item - format_.print_newline(); - format_.print_indents(depth + 1); - items_on_line = 0; - } else if (options_.comma_padding) { - format_.print_space(); - } - } - first = false; - - // Format element inline - layout_mode prev_layout = format_.get_layout_mode(); - format_.set_layout_mode(layout_mode::INLINE); - const element_metrics& child_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; - format_element(elem, child_metrics, depth + 1); - format_.set_layout_mode(prev_layout); - - items_on_line++; - child_idx++; - } - - format_.print_newline(); - format_.print_indents(depth); - format_.end_array(); -} - -inline void fractured_string_builder::format_array_as_table(const dom::array& arr, - const element_metrics& metrics, - size_t depth) { - const std::vector& columns = metrics.common_keys; - if (columns.empty()) { - format_array_expanded(arr, metrics, depth); - return; - } - - // Calculate column widths for alignment - std::vector col_widths = calculate_column_widths(arr, columns); - format_.set_column_widths(col_widths); - - format_.start_array(); - format_.print_newline(); - - bool first_row = true; - size_t child_idx = 0; - for (dom::element elem : arr) { - if (!first_row) { - format_.comma(); - format_.print_newline(); - } - first_row = false; - - format_.print_indents(depth + 1); - format_.begin_table_row(); - - // Format object as inline with aligned columns - dom::object obj; - if (elem.get_object().get(obj) != SUCCESS) { - child_idx++; - continue; - } - - // Get child metrics for this row (object) - const element_metrics& row_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; - - format_.start_object(); - if (options_.simple_bracket_padding) { - format_.print_space(); - } - - bool first_col = true; - const size_t num_columns = columns.size(); - - for (size_t col_idx = 0; col_idx < num_columns; col_idx++) { - const std::string& key = columns[col_idx]; - const bool is_last_col = (col_idx == num_columns - 1); - - if (!first_col) { - format_.comma(); - if (options_.comma_padding) { - format_.print_space(); - } - } - first_col = false; - - // Write key - format_.key(key); - if (options_.colon_padding) { - format_.print_space(); - } - - // Find the value for this key and its metrics - dom::element value; - bool found = false; - size_t field_idx = 0; - for (dom::key_value_pair field : obj) { - if (field.key == key) { - value = field.value; - found = true; - break; - } - field_idx++; - } - - // Write value - if (found) { - layout_mode prev_layout = format_.get_layout_mode(); - format_.set_layout_mode(layout_mode::INLINE); - const element_metrics& value_metrics = (field_idx < row_metrics.children.size()) - ? row_metrics.children[field_idx] : element_metrics{}; - format_element(value, value_metrics, depth + 1); - format_.set_layout_mode(prev_layout); - } else { - format_.null_atom(); - } - - // Only pad non-last columns to align values across rows - if (!is_last_col) { - size_t actual_len = found ? measure_value_length(value) : 4; // 4 for "null" - size_t target_width = col_widths[col_idx]; - while (actual_len < target_width) { - format_.one_char(' '); - actual_len++; - } - } - - format_.next_column(); - } - - if (options_.simple_bracket_padding) { - format_.print_space(); - } - format_.end_object(); - format_.end_table_row(); - child_idx++; - } - - format_.print_newline(); - format_.print_indents(depth); - format_.end_array(); -} - -inline void fractured_string_builder::format_array_expanded(const dom::array& arr, - const element_metrics& metrics, - size_t depth) { - format_.start_array(); - - bool empty = true; - bool first = true; - size_t child_idx = 0; - - for (dom::element elem : arr) { - empty = false; - if (!first) { - format_.comma(); - } - first = false; - - format_.print_newline(); - format_.print_indents(depth + 1); - const element_metrics& child_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; - format_element(elem, child_metrics, depth + 1); - child_idx++; - } - - if (!empty) { - format_.print_newline(); - format_.print_indents(depth); - } - format_.end_array(); -} - -inline void fractured_string_builder::format_object(const dom::object& obj, - const element_metrics& metrics, - size_t depth) { - if (metrics.recommended_layout == layout_mode::INLINE || metrics.can_inline) { - format_object_inline(obj, metrics); - } else { - format_object_expanded(obj, metrics, depth); - } -} - -inline void fractured_string_builder::format_object_inline(const dom::object& obj, - const element_metrics& metrics) { - layout_mode prev_layout = format_.get_layout_mode(); - format_.set_layout_mode(layout_mode::INLINE); - - format_.start_object(); - - bool empty = true; - bool first = true; - size_t child_idx = 0; - - for (dom::key_value_pair field : obj) { - empty = false; - if (!first) { - format_.comma(); - if (options_.comma_padding) { - format_.print_space(); - } - } else if (options_.simple_bracket_padding) { - format_.print_space(); - } - first = false; - - format_.key(field.key); - if (options_.colon_padding) { - format_.print_space(); - } - const element_metrics& child_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; - format_element(field.value, child_metrics, 0); - child_idx++; - } - - if (options_.simple_bracket_padding && !empty) { - format_.print_space(); - } - format_.end_object(); - - format_.set_layout_mode(prev_layout); -} - -inline void fractured_string_builder::format_object_expanded(const dom::object& obj, - const element_metrics& metrics, - size_t depth) { - format_.start_object(); - - bool empty = true; - bool first = true; - size_t child_idx = 0; - - for (dom::key_value_pair field : obj) { - empty = false; - if (!first) { - format_.comma(); - } - first = false; - - format_.print_newline(); - format_.print_indents(depth + 1); - format_.key(field.key); - if (options_.colon_padding) { - format_.print_space(); - } - const element_metrics& child_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; - format_element(field.value, child_metrics, depth + 1); - child_idx++; - } - - if (!empty) { - format_.print_newline(); - format_.print_indents(depth); - } - format_.end_object(); -} - -inline void fractured_string_builder::format_scalar(const dom::element& elem) { - switch (elem.type()) { - case dom::element_type::STRING: { - std::string_view str; - if (elem.get_string().get(str) == SUCCESS) { - format_.string(str); - } - break; - } - case dom::element_type::INT64: { - int64_t val; - if (elem.get_int64().get(val) == SUCCESS) { - format_.number(val); - } - break; - } - case dom::element_type::UINT64: { - uint64_t val; - if (elem.get_uint64().get(val) == SUCCESS) { - format_.number(val); - } - break; - } - case dom::element_type::DOUBLE: { - double val; - if (elem.get_double().get(val) == SUCCESS) { - format_.number(val); - } - break; - } - case dom::element_type::BOOL: { - bool val; - if (elem.get_bool().get(val) == SUCCESS) { - val ? format_.true_atom() : format_.false_atom(); - } - break; - } - case dom::element_type::NULL_VALUE: - format_.null_atom(); - break; - default: - break; - } -} - -inline size_t fractured_string_builder::measure_value_length(const dom::element& elem) const { - switch (elem.type()) { - case dom::element_type::STRING: { - std::string_view str; - if (elem.get_string().get(str) == SUCCESS) { - // Count actual escaped length - size_t len = 2; // quotes - for (char c : str) { - if (c == '"' || c == '\\' || static_cast(c) < 32) { - len += 2; // escape sequence - } else { - len += 1; - } - } - return len; - } - return 2; - } - case dom::element_type::INT64: { - int64_t val; - if (elem.get_int64().get(val) == SUCCESS) { - if (val == 0) return 1; - // Handle INT64_MIN specially to avoid overflow when negating - if (val == INT64_MIN) return 20; // "-9223372036854775808" is 20 characters - size_t len = (val < 0) ? 1 : 0; - int64_t abs_val = (val < 0) ? -val : val; - while (abs_val > 0) { len++; abs_val /= 10; } - return len; - } - return 1; - } - case dom::element_type::UINT64: { - uint64_t val; - if (elem.get_uint64().get(val) == SUCCESS) { - if (val == 0) return 1; - size_t len = 0; - while (val > 0) { len++; val /= 10; } - return len; - } - return 1; - } - case dom::element_type::DOUBLE: { - double val; - if (elem.get_double().get(val) == SUCCESS) { - char buf[32]; - int len = snprintf(buf, sizeof(buf), "%.17g", val); - return len > 0 ? static_cast(len) : 1; - } - return 1; - } - case dom::element_type::BOOL: { - bool val; - if (elem.get_bool().get(val) == SUCCESS) { - return val ? 4 : 5; // "true" or "false" - } - return 5; - } - case dom::element_type::NULL_VALUE: - return 4; // "null" - default: - return 4; - } -} - -inline std::vector fractured_string_builder::calculate_column_widths( - const dom::array& arr, - const std::vector& columns) const { - - std::vector widths(columns.size(), 0); - - for (dom::element elem : arr) { - dom::object obj; - if (elem.get_object().get(obj) != SUCCESS) { - continue; - } - - for (size_t col_idx = 0; col_idx < columns.size(); col_idx++) { - const std::string& key = columns[col_idx]; - - for (dom::key_value_pair field : obj) { - if (field.key == key) { - // Measure actual value length - size_t len = measure_value_length(field.value); - widths[col_idx] = (std::max)(widths[col_idx], len); - break; - } - } - } - } - - return widths; -} - -} // namespace internal - -// -// Public API Implementation -// - -template -std::string fractured_json(T x) { - return fractured_json(x, fractured_json_options{}); -} - -template -std::string fractured_json(T x, const fractured_json_options& options) { - internal::fractured_string_builder sb(options); - sb.append(x); - std::string_view result = sb.str(); - return std::string(result.data(), result.size()); -} - -#if SIMDJSON_EXCEPTIONS -template -std::string fractured_json(simdjson_result x) { - if (x.error()) { - throw simdjson_error(x.error()); - } - return fractured_json(x.value()); -} - -template -std::string fractured_json(simdjson_result x, const fractured_json_options& options) { - if (x.error()) { - throw simdjson_error(x.error()); - } - return fractured_json(x.value(), options); -} -#endif - -// Explicit template instantiations for common types -template std::string fractured_json(dom::element x); -template std::string fractured_json(dom::element x, const fractured_json_options& options); -template std::string fractured_json(dom::array x); -template std::string fractured_json(dom::array x, const fractured_json_options& options); -template std::string fractured_json(dom::object x); -template std::string fractured_json(dom::object x, const fractured_json_options& options); - -#if SIMDJSON_EXCEPTIONS -template std::string fractured_json(simdjson_result x); -template std::string fractured_json(simdjson_result x, const fractured_json_options& options); -#endif - -// -// String-based API for formatting any JSON string -// - -inline std::string fractured_json_string(std::string_view json_str) { - return fractured_json_string(json_str, fractured_json_options{}); -} - -inline std::string fractured_json_string(std::string_view json_str, - const fractured_json_options& options) { - // Parse the JSON string - dom::parser parser; - dom::element doc; - // Need to pad the string for simdjson - auto padded = padded_string(json_str); - auto error = parser.parse(padded).get(doc); - if (error) { - // If parsing fails, return the original string - return std::string(json_str); - } - return fractured_json(doc, options); -} - -} // namespace simdjson - -#endif // SIMDJSON_DOM_FRACTURED_JSON_INL_H -/* end file simdjson/dom/fractured_json-inl.h */ - -#endif // SIMDJSON_DOM_H -/* end file simdjson/dom.h */ -/* including simdjson/builder.h: #include "simdjson/builder.h" */ -/* begin file simdjson/builder.h */ -#ifndef SIMDJSON_BUILDER_H -#define SIMDJSON_BUILDER_H - -/* including simdjson/builtin/builder.h: #include "simdjson/builtin/builder.h" */ -/* begin file simdjson/builtin/builder.h */ -#ifndef SIMDJSON_BUILTIN_BUILDER_H -#define SIMDJSON_BUILTIN_BUILDER_H - /* including simdjson/builtin.h: #include "simdjson/builtin.h" */ /* begin file simdjson/builtin.h */ #ifndef SIMDJSON_BUILTIN_H @@ -38079,6 +29854,8404 @@ simdjson_inline implementation_simdjson_result_base::implementation_simdjson_ #endif // SIMDJSON_BUILTIN_H /* end file simdjson/builtin.h */ +/* including simdjson/minify.h: #include "simdjson/minify.h" */ +/* begin file simdjson/minify.h */ +#ifndef SIMDJSON_MINIFY_H +#define SIMDJSON_MINIFY_H + +/* skipped duplicate #include "simdjson/base.h" */ +/* including simdjson/padded_string.h: #include "simdjson/padded_string.h" */ +/* begin file simdjson/padded_string.h */ +#ifndef SIMDJSON_PADDED_STRING_H +#define SIMDJSON_PADDED_STRING_H + +/* skipped duplicate #include "simdjson/base.h" */ +/* skipped duplicate #include "simdjson/error.h" */ + +/* skipped duplicate #include "simdjson/error-inl.h" */ + +#include +#include +#include +#include + +namespace simdjson { + +class padded_string_view; + +/** + * String with extra allocation for ease of use with parser::parse() + * + * This is a move-only class, it cannot be copied. + */ +struct padded_string final { + + /** + * Create a new, empty padded string. + */ + explicit inline padded_string() noexcept; + /** + * Create a new padded string buffer. + * + * @param length the size of the string. + */ + explicit inline padded_string(size_t length) noexcept; + /** + * Create a new padded string by copying the given input. + * + * @param data the buffer to copy + * @param length the number of bytes to copy + */ + explicit inline padded_string(const char *data, size_t length) noexcept; +#ifdef __cpp_char8_t + explicit inline padded_string(const char8_t *data, size_t length) noexcept; +#endif + /** + * Create a new padded string by copying the given input. + * + * @param str_ the string to copy + */ + inline padded_string(const std::string & str_ ) noexcept; + /** + * Create a new padded string by copying the given input. + * + * @param sv_ the string to copy + */ + inline padded_string(std::string_view sv_) noexcept; + /** + * Move one padded string into another. + * + * The original padded string will be reduced to zero capacity. + * + * @param o the string to move. + */ + inline padded_string(padded_string &&o) noexcept; + /** + * Move one padded string into another. + * + * The original padded string will be reduced to zero capacity. + * + * @param o the string to move. + */ + inline padded_string &operator=(padded_string &&o) noexcept; + inline void swap(padded_string &o) noexcept; + ~padded_string() noexcept; + + /** + * The length of the string. + * + * Does not include padding. + */ + size_t size() const noexcept; + + /** + * The length of the string. + * + * Does not include padding. + */ + size_t length() const noexcept; + + /** + * The string data. + **/ + const char *data() const noexcept; + const uint8_t *u8data() const noexcept { return static_cast(static_cast(data_ptr));} + + /** + * The string data. + **/ + char *data() noexcept; + + /** + * Append data to the padded string. Return true on success, false on failure. + * The complexity is O(n) where n is the new size of the string. If you are + * doing multiple appends, consider using padded_string_builder for better performance. + * + * @param data the buffer to append + * @param length the number of bytes to append + */ + inline bool append(const char *data, size_t length) noexcept; + + /** + * Create a std::string_view with the same content. + */ + operator std::string_view() const; + + /** + * Create a padded_string_view with the same content. + */ + operator padded_string_view() const noexcept; + + /** + * Load this padded string from a file. + * + * ## Windows and Unicode + * + * Windows users who need to read files with non-ANSI characters in the + * name should set their code page to UTF-8 (65001) before calling this + * function. This should be the default with Windows 11 and better. + * Further, they may use the AreFileApisANSI function to determine whether + * the filename is interpreted using the ANSI or the system default OEM + * codepage, and they may call SetFileApisToOEM accordingly. + * + * @return IO_ERROR on error. Be mindful that on some 32-bit systems, + * the file size might be limited to 2 GB. + * + * @param path the path to the file. + **/ + inline static simdjson_result load(std::string_view path) noexcept; + + #if defined(_WIN32) && SIMDJSON_CPLUSPLUS17 + /** + * This function accepts a wide string path (UTF-16) and converts it to + * UTF-8 before loading the file. This allows windows users to work + * with unicode file paths without manually converting the paths every time. + * + * @return IO_ERROR on error, including conversion failures. + * + * @param path the path to the file as a wide string. + **/ + inline static simdjson_result load(std::wstring_view path) noexcept; + #endif + +private: + friend class padded_string_builder; + padded_string &operator=(const padded_string &o) = delete; + padded_string(const padded_string &o) = delete; + + size_t viable_size{0}; + char *data_ptr{nullptr}; + +}; // padded_string + +/** + * Builder for constructing padded_string incrementally. + * + * This class allows efficient appending of data and then building a padded_string. + */ +class padded_string_builder { +public: + /** + * Create a new, empty padded string builder. + */ + inline padded_string_builder() noexcept; + + /** + * Create a new padded string builder with initial capacity. + * + * @param capacity the initial capacity of the builder. + */ + inline padded_string_builder(size_t capacity) noexcept; + + /** + * Move constructor. + */ + inline padded_string_builder(padded_string_builder &&o) noexcept; + + /** + * Move assignment. + */ + inline padded_string_builder &operator=(padded_string_builder &&o) noexcept; + + /** + * Copy constructor (deleted). + */ + padded_string_builder(const padded_string_builder &) = delete; + + /** + * Copy assignment (deleted). + */ + padded_string_builder &operator=(const padded_string_builder &) = delete; + + /** + * Destructor. + */ + inline ~padded_string_builder() noexcept; + + /** + * Append data to the builder. + * + * @param newdata the buffer to append + * @param length the number of bytes to append + * @return true if the append succeeded, false if allocation failed + */ + inline bool append(const char *newdata, size_t length) noexcept; + + /** + * Append a string view to the builder. + * + * @param sv the string view to append + * @return true if the append succeeded, false if allocation failed + */ + inline bool append(std::string_view sv) noexcept; + + /** + * Get the current length of the built string. + */ + inline size_t length() const noexcept; + + /** + * Build a padded_string from the current content. The builder's content + * is not modified. If you want to avoid the copy, use convert() instead. + * + * @return a padded_string containing a copy of the built content. + */ + inline padded_string build() const noexcept; + + /** + * Convert the current content into a padded_string. The + * builder's content is emptied, the capacity is lost. + * + * @return a padded_string containing the built content. + */ + inline padded_string convert() noexcept; +private: + size_t size{0}; + size_t capacity{0}; + char *data{nullptr}; + + /** + * Ensure the builder has enough capacity. + * + * @param additional the additional capacity needed. + * @return true if the reservation succeeded, false if allocation failed + */ + inline bool reserve(size_t additional) noexcept; +}; + +/** + * Send padded_string instance to an output stream. + * + * @param out The output stream. + * @param s The padded_string instance. + * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + */ +inline std::ostream& operator<<(std::ostream& out, const padded_string& s) { return out << s.data(); } + +#if SIMDJSON_EXCEPTIONS +/** + * Send padded_string instance to an output stream. + * + * @param out The output stream. + * @param s The padded_string instance. + * @throw simdjson_error if the result being printed has an error. If there is an error with the + * underlying output stream, that error will be propagated (simdjson_error will not be + * thrown). + */ +inline std::ostream& operator<<(std::ostream& out, simdjson_result &s) noexcept(false) { return out << s.value(); } +#endif + + +#ifndef _WIN32 +/** + * A class representing a memory-mapped file with padding. + * It is only available on non-Windows platforms, as Windows has different APIs for memory mapping. + */ +class padded_memory_map { +public: + /** + * Create a new padded memory map for the given file. + * After creating the memory map, you can call view() to get a padded_string_view of the file content. + * The memory map will be automatically released when the padded_memory_map instance is destroyed. + * Note that the file content is not copied, so this is efficient for large files. However, + * the file must remain unchanged while the memory map is in use. In case of error (e.g., file not found, + * permission denied, etc.), the memory map will be invalid and view() will return an empty view. + * You can check if the memory map is valid by calling is_valid() before using view(). + * + * @param filename the path to the file to memory-map. + */ + simdjson_inline padded_memory_map(const char *filename) noexcept; + /** + * Destroy the padded memory map and release any resources. + */ + simdjson_inline ~padded_memory_map() noexcept; + + // lifetime of the view is tied to the memory map, so we can return a view + // directly + /** + * Get a view of the memory-mapped file. It always succeeds, but the view may be empty + * if the memory map is invalid (e.g., due to file not found, permission denied, etc.). + * You can check if the memory map is valid by calling is_valid() before using the view. + * + * Lifetime of the view is tied to the memory map, so the view should not be used after the + * padded_memory_map instance is destroyed. + * + * @return a padded_string_view representing the memory-mapped file, or an empty view if the memory map is invalid. + */ + simdjson_inline simdjson::padded_string_view view() const noexcept simdjson_lifetime_bound; + /** + * Check if the memory map is valid. + * + * @return true if the memory map is valid, false otherwise. + */ + simdjson_inline bool is_valid() const noexcept; + +private: + padded_memory_map() = delete; + padded_memory_map(const padded_memory_map &) = delete; + padded_memory_map &operator=(const padded_memory_map &) = delete; + const char *data{nullptr}; + size_t size{0}; +}; +#endif // _WIN32 + + + +} // namespace simdjson + +// This is deliberately outside of simdjson so that people get it without having to use the namespace +inline simdjson::padded_string operator ""_padded(const char *str, size_t len); +#ifdef __cpp_char8_t +inline simdjson::padded_string operator ""_padded(const char8_t *str, size_t len); +#endif + +namespace simdjson { +namespace internal { + +// The allocate_padded_buffer function is a low-level function to allocate memory +// with padding so we can read past the "length" bytes safely. It is used by +// the padded_string class automatically. It returns nullptr in case +// of error: the caller should check for a null pointer. +// The length parameter is the maximum size in bytes of the string. +// The caller is responsible to free the memory (e.g., delete[] (...)). +inline char *allocate_padded_buffer(size_t length) noexcept; + +} // namespace internal +} // namespace simdjson + +#endif // SIMDJSON_PADDED_STRING_H +/* end file simdjson/padded_string.h */ +#include +#include +#include + +namespace simdjson { + +/** + * + * Minify the input string assuming that it represents a JSON string, does not parse or validate. + * This function is much faster than parsing a JSON string and then writing a minified version of it. + * However, it does not validate the input. It will merely return an error in simple cases (e.g., if + * there is a string that was never terminated). + * + * + * @param buf the json document to minify. + * @param len the length of the json document. + * @param dst the buffer to write the minified document to. *MUST* be allocated up to len bytes. + * @param dst_len the number of bytes written. Output only. + * @return the error code, or SUCCESS if there was no error. + */ +simdjson_warn_unused error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept; + +} // namespace simdjson + +#endif // SIMDJSON_MINIFY_H +/* end file simdjson/minify.h */ +/* skipped duplicate #include "simdjson/padded_string.h" */ +/* including simdjson/padded_string-inl.h: #include "simdjson/padded_string-inl.h" */ +/* begin file simdjson/padded_string-inl.h */ +#ifndef SIMDJSON_PADDED_STRING_INL_H +#define SIMDJSON_PADDED_STRING_INL_H + +/* skipped duplicate #include "simdjson/padded_string.h" */ +/* including simdjson/padded_string_view.h: #include "simdjson/padded_string_view.h" */ +/* begin file simdjson/padded_string_view.h */ +#ifndef SIMDJSON_PADDED_STRING_VIEW_H +#define SIMDJSON_PADDED_STRING_VIEW_H + +/* skipped duplicate #include "simdjson/portability.h" */ +/* skipped duplicate #include "simdjson/base.h" // for SIMDJSON_PADDING */ +/* skipped duplicate #include "simdjson/error.h" */ + +#include +#include +#include +#include +#if SIMDJSON_CPLUSPLUS17 +#include +#endif + +namespace simdjson { + +/** + * User-provided string that promises it has extra padded bytes at the end for use with parser::parse(). + */ +class padded_string_view : public std::string_view { +private: + size_t _capacity{0}; + +public: + /** Create an empty padded_string_view. */ + inline padded_string_view() noexcept = default; + + /** + * Promise the given buffer has at least SIMDJSON_PADDING extra bytes allocated to it. + * + * @param s The string. + * @param len The length of the string (not including padding). + * @param capacity The allocated length of the string, including padding. If the capacity is less + * than the length, the capacity will be set to the length. + */ + explicit inline padded_string_view(const char* s, size_t len, size_t capacity) noexcept; + /** overload explicit inline padded_string_view(const char* s, size_t len) noexcept */ + explicit inline padded_string_view(const uint8_t* s, size_t len, size_t capacity) noexcept; +#ifdef __cpp_char8_t + explicit inline padded_string_view(const char8_t* s, size_t len, size_t capacity) noexcept; +#endif + /** + * Promise the given string has at least SIMDJSON_PADDING extra bytes allocated to it. + * + * The capacity of the string will be used to determine its padding. + * + * @param s The string. + */ + explicit inline padded_string_view(const std::string &s) noexcept; + + /** + * Promise the given string_view has at least SIMDJSON_PADDING extra bytes allocated to it. + * + * @param s The string. + * @param capacity The allocated length of the string, including padding. If the capacity is less + * than the length, the capacity will be set to the length. + */ + explicit inline padded_string_view(std::string_view s, size_t capacity) noexcept; + + /** The number of allocated bytes. */ + inline size_t capacity() const noexcept; + + /** check that the view has sufficient padding */ + inline bool has_sufficient_padding() const noexcept; + + /** + * Remove the UTF-8 Byte Order Mark (BOM) if it exists. + * + * @return whether a BOM was found and removed + */ + inline bool remove_utf8_bom() noexcept; + + /** The amount of padding on the string (capacity() - length()) */ + inline size_t padding() const noexcept; + +}; // padded_string_view + +/** + * Get the system's memory page size. By default, we return + * 4096 bytes, which is the most common page size. On systems + * where the page size is not a multiple of 4096 bytes, and not + * a unix-like system, nor Windows, this function may return an + * incorrect value. + * + * @return The page size in bytes. + */ +inline uint32_t get_page_size() noexcept; + +#if SIMDJSON_CPLUSPLUS17 +/** + * A padded_input is a wrapper around either a padded_string_view or a padded_string. + * It will automatically pad a string_view if it does not have sufficient padding + * up to the end of the memory page. Note that a requirement for this method to + * make sense is to be on a system with a page size of at least 4096 (which is + * universal except on some embedded systems). + */ +struct padded_input { + /** + * Construct a padded_input from a string_view. If the string_view does not have sufficient padding, + * the data will be copied into a padded_string and the padded_string_view will point to the + * padded_string's data. Otherwise, the padded_string_view will point to the original string_view's data. + */ + inline explicit padded_input(std::string_view sv); + /** + * Construct a padded_input from a C-style string (length specified). If the string does not have sufficient padding, + * the data will be copied into a padded_string and the padded_string_view will point to the + * padded_string's data. Otherwise, the padded_string_view will point to the original string's data. + */ + inline explicit padded_input(const char *data, size_t length); + /** + * Construct a padded_input from a std::string. If the string does not have sufficient padding + * (considering its capacity), the data will be copied into a padded_string and the padded_string_view + * will point to the padded_string's data. Otherwise, the padded_string_view will point to the + * original string's data. + */ + inline explicit padded_input(const std::string &s); + + /** + * Check if the padded_input is a view. + * + * @return true if the padded_input is a view, false otherwise. + */ + inline bool is_view() const noexcept; + + /** + * Convert the padded_input to a padded_string_view. + * + * @return The padded_string_view. + */ + inline operator simdjson::padded_string_view() const noexcept; + +private: + std::variant storage; + // whether we cross a page boundary and need to allocate a new padded string. + static inline bool needs_allocation(const char* buf, size_t len, size_t padding = SIMDJSON_PADDING) noexcept; +}; + +#endif // SIMDJSON_CPLUSPLUS17 + +#if SIMDJSON_EXCEPTIONS +/** + * Send padded_string instance to an output stream. + * + * @param out The output stream. + * @param s The padded_string_view. + * @throw simdjson_error if the result being printed has an error. If there is an error with the + * underlying output stream, that error will be propagated (simdjson_error will not be + * thrown). + */ +inline std::ostream& operator<<(std::ostream& out, simdjson_result &s) noexcept(false); +#endif + +/** + * Create a padded_string_view from a string. The string will be padded with up to SIMDJSON_PADDING + * space characters. The resulting padded_string_view will have a length equal to the original + * string, except maybe for trailing white space characters. + * + * @param s The string. + * @return The padded string. + */ +inline padded_string_view pad(std::string& s) noexcept; + +/** + * Create a padded_string_view from a string. The capacity of the string will be padded with SIMDJSON_PADDING + * characters. The resulting padded_string_view will have a length equal to the original + * string. + * + * @param s The string. + * @return The padded string. + */ +inline padded_string_view pad_with_reserve(std::string& s) noexcept; + +} // namespace simdjson + +#endif // SIMDJSON_PADDED_STRING_VIEW_H +/* end file simdjson/padded_string_view.h */ + +/* skipped duplicate #include "simdjson/error-inl.h" */ +/* including simdjson/padded_string_view-inl.h: #include "simdjson/padded_string_view-inl.h" */ +/* begin file simdjson/padded_string_view-inl.h */ +#ifndef SIMDJSON_PADDED_STRING_VIEW_INL_H +#define SIMDJSON_PADDED_STRING_VIEW_INL_H + +/* skipped duplicate #include "simdjson/padded_string_view.h" */ +/* skipped duplicate #include "simdjson/error-inl.h" */ + +#include /* memcmp */ + +// for page size computation. +#if defined(__unix__) || defined(__APPLE__) || defined(__linux__) + #include + #if defined(__APPLE__) + #include + #endif +#endif + + +namespace simdjson { + +inline padded_string_view::padded_string_view(const char* s, size_t len, size_t capacity) noexcept + : std::string_view(s, len), _capacity(capacity) +{ + if(_capacity < len) { _capacity = len; } +} + +inline padded_string_view::padded_string_view(const uint8_t* s, size_t len, size_t capacity) noexcept + : padded_string_view(reinterpret_cast(s), len, capacity) +{ +} +#ifdef __cpp_char8_t +inline padded_string_view::padded_string_view(const char8_t* s, size_t len, size_t capacity) noexcept + : padded_string_view(reinterpret_cast(s), len, capacity) +{ +} +#endif +inline padded_string_view::padded_string_view(const std::string &s) noexcept + : std::string_view(s), _capacity(s.capacity()) +{ +} + +inline padded_string_view::padded_string_view(std::string_view s, size_t capacity) noexcept + : std::string_view(s), _capacity(capacity) +{ + if(_capacity < s.length()) { _capacity = s.length(); } +} + +inline bool padded_string_view::has_sufficient_padding() const noexcept { + if (padding() >= SIMDJSON_PADDING) { + return true; + } + size_t missing_padding = SIMDJSON_PADDING - padding(); + if(length() < missing_padding) { return false; } + + for (size_t i = length() - missing_padding; i < length(); i++) { + char c = data()[i]; + if (c != ' ' && c != '\t' && c != '\n' && c != '\r') { + return false; + } + } + return true; +} + +inline size_t padded_string_view::capacity() const noexcept { return _capacity; } + +inline size_t padded_string_view::padding() const noexcept { return capacity() - length(); } + +inline bool padded_string_view::remove_utf8_bom() noexcept { + if(length() < 3) { return false; } + if (std::memcmp(data(), "\xEF\xBB\xBF", 3) == 0) { + remove_prefix(3); + _capacity -= 3; + return true; + } + return false; +} + +#if SIMDJSON_EXCEPTIONS +inline std::ostream& operator<<(std::ostream& out, simdjson_result &s) noexcept(false) { return out << s.value(); } +#endif + +inline padded_string_view pad(std::string& s) noexcept { + size_t existing_padding = 0; + for (size_t i = s.size(); i > 0; i--) { + char c = s[i - 1]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + existing_padding++; + } else { + break; + } + } + size_t needed_padding = 0; + if (existing_padding < SIMDJSON_PADDING) { + needed_padding = SIMDJSON_PADDING - existing_padding; + s.append(needed_padding, ' '); + } + + return padded_string_view(s.data(), s.size() - needed_padding, s.size()); +} + +inline padded_string_view pad_with_reserve(std::string& s) noexcept { + if (s.capacity() - s.size() < SIMDJSON_PADDING) { + s.reserve(s.size() + SIMDJSON_PADDING ); + } + return padded_string_view(s.data(), s.size(), s.capacity()); +} + +inline uint32_t get_page_size() noexcept { +#if defined(_WINDOWS_) // if and only if someone loaded Windows.h, we can get the page size from there. +// Otherwise, we assume 4096. + static const uint32_t cached = []() -> uint32_t { + SYSTEM_INFO si; + GetSystemInfo(&si); + return static_cast(si.dwPageSize); + }(); + return cached; +#elif defined(__unix__) || defined(__APPLE__) || defined(__linux__) + static const uint32_t cached = []() -> uint32_t { + long page_size = sysconf(_SC_PAGESIZE); + if (page_size > 0) { + return static_cast(page_size); + } + return 4096; // fallback + }(); + return cached; +#else + return 4096; // fallback +#endif +} +#if SIMDJSON_CPLUSPLUS17 + +inline padded_input::padded_input(std::string_view sv) + : storage(simdjson::padded_string_view{}) { + if (needs_allocation(sv.data(), sv.size())) { + storage = simdjson::padded_string(sv); + } else { + storage = simdjson::padded_string_view( + sv.data(), sv.size(), sv.size() + simdjson::SIMDJSON_PADDING); + } +} + +inline padded_input::padded_input(const char *data, size_t length) + : storage(simdjson::padded_string_view{}) { + if (needs_allocation(data, length)) { + storage = simdjson::padded_string(data, length); + } else { + storage = simdjson::padded_string_view( + data, length, length + simdjson::SIMDJSON_PADDING); + } +} + +inline padded_input::padded_input(const std::string &s) + : storage(simdjson::padded_string_view{}) { + const size_t len = s.size(); + const size_t cap = s.capacity(); + // Here we have the string content from data() to data() + size(), + // but the memory is accessible from data() to data() + capacity(). + const size_t needed_padding = (cap - len) < simdjson::SIMDJSON_PADDING + ? simdjson::SIMDJSON_PADDING - (cap - len) : 0; + if (needed_padding > 0 && needs_allocation(s.data(), cap, needed_padding)) { + storage = simdjson::padded_string(s); + } else { + storage = simdjson::padded_string_view( + s.data(), len, len + simdjson::SIMDJSON_PADDING); + } +} + +inline bool padded_input::is_view() const noexcept { + return std::holds_alternative(storage); +} + +inline padded_input::operator simdjson::padded_string_view() const noexcept { + return std::visit([](const auto& p) -> simdjson::padded_string_view { + return p; + }, storage); +} + +inline bool padded_input::needs_allocation(const char* buf, size_t len, size_t padding) noexcept { + if(len == 0) { return false; } + const auto page_size = get_page_size(); + return ((reinterpret_cast(buf + len - 1) % page_size) + + padding >= static_cast(page_size)); +} +#endif // SIMDJSON_CPLUSPLUS17 + +} // namespace simdjson + + +#endif // SIMDJSON_PADDED_STRING_VIEW_INL_H +/* end file simdjson/padded_string_view-inl.h */ + +#include +#include + +#ifndef _WIN32 +#include +#include +#include +#include +#include +#endif + +namespace simdjson { +namespace internal { + +// The allocate_padded_buffer function is a low-level function to allocate memory +// with padding so we can read past the "length" bytes safely. It is used by +// the padded_string class automatically. It returns nullptr in case +// of error: the caller should check for a null pointer. +// The length parameter is the maximum size in bytes of the string. +// The caller is responsible to free the memory (e.g., delete[] (...)). +inline char *allocate_padded_buffer(size_t length) noexcept { + const size_t totalpaddedlength = length + SIMDJSON_PADDING; + if(totalpaddedlength(1UL<<20)) { + return nullptr; + } +#endif + + char *padded_buffer = new (std::nothrow) char[totalpaddedlength]; + if (padded_buffer == nullptr) { + return nullptr; + } + // We write nulls in the padded region to avoid having uninitialized + // content which may trigger warning for some sanitizers + std::memset(padded_buffer + length, 0, totalpaddedlength - length); + return padded_buffer; +} // allocate_padded_buffer() + +} // namespace internal + + +inline padded_string::padded_string() noexcept = default; +inline padded_string::padded_string(size_t length) noexcept + : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { +} +inline padded_string::padded_string(const char *data, size_t length) noexcept + : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { + if ((data != nullptr) && (data_ptr != nullptr)) { + std::memcpy(data_ptr, data, length); + } + if (data_ptr == nullptr) { + viable_size = 0; + } +} +#ifdef __cpp_char8_t +inline padded_string::padded_string(const char8_t *data, size_t length) noexcept + : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { + if ((data != nullptr) && (data_ptr != nullptr)) { + std::memcpy(data_ptr, reinterpret_cast(data), length); + } + if (data_ptr == nullptr) { + viable_size = 0; + } +} +#endif +// note: do not pass std::string arguments by value +inline padded_string::padded_string(const std::string & str_ ) noexcept + : viable_size(str_.size()), data_ptr(internal::allocate_padded_buffer(str_.size())) { + if (data_ptr == nullptr) { + viable_size = 0; + } else { + std::memcpy(data_ptr, str_.data(), str_.size()); + } +} +// note: do pass std::string_view arguments by value +inline padded_string::padded_string(std::string_view sv_) noexcept + : viable_size(sv_.size()), data_ptr(internal::allocate_padded_buffer(sv_.size())) { + if(simdjson_unlikely(!data_ptr)) { + //allocation failed or zero size + viable_size = 0; + return; + } + if (sv_.size()) { + std::memcpy(data_ptr, sv_.data(), sv_.size()); + } +} +inline padded_string::padded_string(padded_string &&o) noexcept + : viable_size(o.viable_size), data_ptr(o.data_ptr) { + o.data_ptr = nullptr; // we take ownership + o.viable_size = 0; +} + +inline padded_string &padded_string::operator=(padded_string &&o) noexcept { + delete[] data_ptr; + data_ptr = o.data_ptr; + viable_size = o.viable_size; + o.data_ptr = nullptr; // we take ownership + o.viable_size = 0; + return *this; +} + +inline void padded_string::swap(padded_string &o) noexcept { + size_t tmp_viable_size = viable_size; + char *tmp_data_ptr = data_ptr; + viable_size = o.viable_size; + data_ptr = o.data_ptr; + o.data_ptr = tmp_data_ptr; + o.viable_size = tmp_viable_size; +} + +inline padded_string::~padded_string() noexcept { + delete[] data_ptr; +} + +inline size_t padded_string::size() const noexcept { return viable_size; } + +inline size_t padded_string::length() const noexcept { return viable_size; } + +inline const char *padded_string::data() const noexcept { return data_ptr; } + +inline char *padded_string::data() noexcept { return data_ptr; } + +inline bool padded_string::append(const char *data, size_t length) noexcept { + if (length == 0) { + return true; // Nothing to append + } + size_t new_size = viable_size + length; + if (new_size < viable_size) { + // Overflow, cannot append + return false; + } + char *new_data_ptr = internal::allocate_padded_buffer(new_size); + if (new_data_ptr == nullptr) { + // Allocation failed, cannot append + return false; + } + // Copy existing data + if (viable_size > 0) { + std::memcpy(new_data_ptr, data_ptr, viable_size); + } + // Copy new data + std::memcpy(new_data_ptr + viable_size, data, length); + // Update + delete[] data_ptr; + data_ptr = new_data_ptr; + viable_size = new_size; + return true; +} + +inline padded_string::operator std::string_view() const simdjson_lifetime_bound { return std::string_view(data(), length()); } + +inline padded_string::operator padded_string_view() const noexcept simdjson_lifetime_bound { + return padded_string_view(data(), length(), length() + SIMDJSON_PADDING); +} + +inline simdjson_result padded_string::load(std::string_view filename) noexcept { + // std::string_view is not guaranteed to be null-terminated, but std::fopen requires + // a null-terminated C string. Construct a temporary std::string to ensure null-termination. + const std::string null_terminated_filename(filename); + // Open the file + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe + std::FILE *fp = std::fopen(null_terminated_filename.c_str(), "rb"); + SIMDJSON_POP_DISABLE_WARNINGS + + if (fp == nullptr) { + return IO_ERROR; + } + + // Get the file size + int ret; +#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS + ret = _fseeki64(fp, 0, SEEK_END); +#else + ret = std::fseek(fp, 0, SEEK_END); +#endif // _WIN64 + if(ret < 0) { + std::fclose(fp); + return IO_ERROR; + } +#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS + __int64 llen = _ftelli64(fp); + if(llen == -1L) { + std::fclose(fp); + return IO_ERROR; + } +#else + long llen = std::ftell(fp); + if((llen < 0) || (llen == LONG_MAX)) { + std::fclose(fp); + return IO_ERROR; + } +#endif + + // Allocate the padded_string + size_t len = static_cast(llen); + padded_string s(len); + if (s.data() == nullptr) { + std::fclose(fp); + return MEMALLOC; + } + + // Read the padded_string + std::rewind(fp); + size_t bytes_read = std::fread(s.data(), 1, len, fp); + if (std::fclose(fp) != 0 || bytes_read != len) { + return IO_ERROR; + } + + return s; +} + +#if defined(_WIN32) && SIMDJSON_CPLUSPLUS17 +inline simdjson_result padded_string::load(std::wstring_view filename) noexcept { + // std::wstring_view is not guaranteed to be null-terminated, but _wfopen requires + // a null-terminated wide C string. Construct a temporary std::wstring to ensure null-termination. + const std::wstring null_terminated_filename(filename); + // Open the file using the wide characters + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe + std::FILE *fp = _wfopen(null_terminated_filename.c_str(), L"rb"); + SIMDJSON_POP_DISABLE_WARNINGS + + if (fp == nullptr) { + return IO_ERROR; + } + + // Get the file size + int ret; +#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS + ret = _fseeki64(fp, 0, SEEK_END); +#else + ret = std::fseek(fp, 0, SEEK_END); +#endif // _WIN64 + if(ret < 0) { + std::fclose(fp); + return IO_ERROR; + } +#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS + __int64 llen = _ftelli64(fp); + if(llen == -1L) { + std::fclose(fp); + return IO_ERROR; + } +#else + long llen = std::ftell(fp); + if((llen < 0) || (llen == LONG_MAX)) { + std::fclose(fp); + return IO_ERROR; + } +#endif + + // Allocate the padded_string + size_t len = static_cast(llen); + padded_string s(len); + if (s.data() == nullptr) { + std::fclose(fp); + return MEMALLOC; + } + + // Read the padded_string + std::rewind(fp); + size_t bytes_read = std::fread(s.data(), 1, len, fp); + if (std::fclose(fp) != 0 || bytes_read != len) { + return IO_ERROR; + } + + return s; +} +#endif + +// padded_string_builder implementations + +inline padded_string_builder::padded_string_builder() noexcept = default; + +inline padded_string_builder::padded_string_builder(size_t new_capacity) noexcept { + if (new_capacity > 0) { + data = internal::allocate_padded_buffer(new_capacity); + if (data != nullptr) { + this->capacity = new_capacity; + } + } +} + +inline padded_string_builder::padded_string_builder(padded_string_builder &&o) noexcept + : size(o.size), capacity(o.capacity), data(o.data) { + o.size = 0; + o.capacity = 0; + o.data = nullptr; +} + +inline padded_string_builder &padded_string_builder::operator=(padded_string_builder &&o) noexcept { + if (this != &o) { + delete[] data; + size = o.size; + capacity = o.capacity; + data = o.data; + o.size = 0; + o.capacity = 0; + o.data = nullptr; + } + return *this; +} + +inline padded_string_builder::~padded_string_builder() noexcept { + delete[] data; +} + +inline bool padded_string_builder::append(const char *newdata, size_t length) noexcept { + if (length == 0) { + return true; + } + if (!reserve(length)) { + return false; + } + std::memcpy(data + size, newdata, length); + size += length; + return true; +} + +inline bool padded_string_builder::append(std::string_view sv) noexcept { + return append(sv.data(), sv.size()); +} + +inline size_t padded_string_builder::length() const noexcept { + return size; +} + +inline padded_string padded_string_builder::build() const noexcept { + return padded_string(data, size); +} + +inline padded_string padded_string_builder::convert() noexcept { + padded_string result{}; + result.data_ptr = data; + result.viable_size = size; + data = nullptr; + size = 0; + capacity = 0; + return result; +} + +inline bool padded_string_builder::reserve(size_t additional) noexcept { + if (simdjson_unlikely(additional + size < size)) { + return false; // overflow: cannot satisfy request + } + size_t needed = size + additional; + if (needed <= capacity) { + return true; + } + size_t new_capacity = needed; + // We are going to grow the capacity exponentially to avoid + // repeated allocations. + if (new_capacity < 4096) { + new_capacity *= 2; + // overflow guard: ensure new_capacity + new_capacity/2 does not overflow + } else if (new_capacity + new_capacity / 2 > new_capacity) { + new_capacity += new_capacity / 2; // grow by 1.5x + } + char *new_data = internal::allocate_padded_buffer(new_capacity); + if (new_data == nullptr) { + return false; // Allocation failed + } + if (size > 0) { + std::memcpy(new_data, data, size); + } + delete[] data; + data = new_data; + capacity = new_capacity; + return true; +} + + +#ifndef _WIN32 +simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept { + + int fd = open(filename, O_RDONLY); + if (fd == -1) { + return; // file not found or cannot be opened, data will be nullptr + } + struct stat st; + if (fstat(fd, &st) == -1) { + close(fd); + return; // failed to get file size, data will be nullptr + } + size = static_cast(st.st_size); + size_t total_size = size + simdjson::SIMDJSON_PADDING; + void *anon_map = + mmap(NULL, total_size, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (anon_map == MAP_FAILED) { + close(fd); + return; // failed to create anonymous mapping, data will be nullptr + } + void *file_map = + mmap(anon_map, size, PROT_READ, MAP_SHARED | MAP_FIXED, fd, 0); + if (file_map == MAP_FAILED) { + munmap(anon_map, total_size); + close(fd); + return; // failed to mmap file, data will be nullptr + } + data = static_cast(file_map); + close(fd); // no longer needed after mapping +} + +simdjson_inline padded_memory_map::~padded_memory_map() noexcept { + if (data != nullptr) { + munmap(const_cast(data), size + simdjson::SIMDJSON_PADDING); + } +} + + +simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noexcept simdjson_lifetime_bound { + if(!is_valid()) { + return simdjson::padded_string_view(); // return an empty view if mapping failed + } + return simdjson::padded_string_view(data, size, size + simdjson::SIMDJSON_PADDING); +} + +simdjson_inline bool padded_memory_map::is_valid() const noexcept { + return data != nullptr; +} +#endif // _WIN32 + +} // namespace simdjson + +inline simdjson::padded_string operator ""_padded(const char *str, size_t len) { + return simdjson::padded_string(str, len); +} +#ifdef __cpp_char8_t +inline simdjson::padded_string operator ""_padded(const char8_t *str, size_t len) { + return simdjson::padded_string(reinterpret_cast(str), len); +} +#endif + +#endif // SIMDJSON_PADDED_STRING_INL_H +/* end file simdjson/padded_string-inl.h */ +/* skipped duplicate #include "simdjson/padded_string_view.h" */ +/* skipped duplicate #include "simdjson/padded_string_view-inl.h" */ + +#if SIMDJSON_FEATURE_DOM_API +/* including simdjson/dom.h: #include "simdjson/dom.h" */ +/* begin file simdjson/dom.h */ +#ifndef SIMDJSON_DOM_H +#define SIMDJSON_DOM_H + +/* including simdjson/dom/base.h: #include "simdjson/dom/base.h" */ +/* begin file simdjson/dom/base.h */ +#ifndef SIMDJSON_DOM_BASE_H +#define SIMDJSON_DOM_BASE_H + +/* skipped duplicate #include "simdjson/base.h" */ + +namespace simdjson { + +/** + * @brief A DOM API on top of the simdjson parser. + */ +namespace dom { + +/** The default batch size for parser.parse_many() and parser.load_many() */ +static constexpr size_t DEFAULT_BATCH_SIZE = 1000000; +/** + * Some adversary might try to set the batch size to 0 or 1, which might cause problems. + * We set a minimum of 32B since anything else is highly likely to be an error. In practice, + * most users will want a much larger batch size. + * + * All non-negative MINIMAL_BATCH_SIZE values should be 'safe' except that, obviously, no JSON + * document can ever span 0 or 1 byte and that very large values would create memory allocation issues. + */ +static constexpr size_t MINIMAL_BATCH_SIZE = 32; + +/** + * It is wasteful to allocate memory for tiny documents (e.g., 4 bytes). + */ +static constexpr size_t MINIMAL_DOCUMENT_CAPACITY = 32; + +class array; +class document; +class document_stream; +class element; +class key_value_pair; +class object; +class parser; + +#ifdef SIMDJSON_THREADS_ENABLED +struct stage1_worker; +#endif // SIMDJSON_THREADS_ENABLED + +} // namespace dom + +namespace internal { + +template +class string_builder; +class tape_ref; + +} // namespace internal + +} // namespace simdjson + +#endif // SIMDJSON_DOM_BASE_H +/* end file simdjson/dom/base.h */ +/* including simdjson/dom/array.h: #include "simdjson/dom/array.h" */ +/* begin file simdjson/dom/array.h */ +#ifndef SIMDJSON_DOM_ARRAY_H +#define SIMDJSON_DOM_ARRAY_H + +#include + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* including simdjson/internal/tape_ref.h: #include "simdjson/internal/tape_ref.h" */ +/* begin file simdjson/internal/tape_ref.h */ +#ifndef SIMDJSON_INTERNAL_TAPE_REF_H +#define SIMDJSON_INTERNAL_TAPE_REF_H + +/* skipped duplicate #include "simdjson/base.h" */ + +namespace simdjson { +namespace dom { +class document; +} // namespace dom + +namespace internal { + +/** + * A reference to an element on the tape. Internal only. + */ +class tape_ref { +public: + simdjson_inline tape_ref() noexcept; + simdjson_inline tape_ref(const dom::document *doc, size_t json_index) noexcept; + inline size_t after_element() const noexcept; + simdjson_inline tape_type tape_ref_type() const noexcept; + simdjson_inline uint64_t tape_value() const noexcept; + simdjson_inline bool is_double() const noexcept; + simdjson_inline bool is_int64() const noexcept; + simdjson_inline bool is_uint64() const noexcept; + simdjson_inline bool is_false() const noexcept; + simdjson_inline bool is_true() const noexcept; + simdjson_inline bool is_null_on_tape() const noexcept;// different name to avoid clash with is_null. + simdjson_inline uint32_t matching_brace_index() const noexcept; + simdjson_inline uint32_t scope_count() const noexcept; + template + simdjson_inline T next_tape_value() const noexcept; + simdjson_inline uint32_t get_string_length() const noexcept; + simdjson_inline const char * get_c_str() const noexcept; + inline std::string_view get_string_view() const noexcept; + simdjson_inline bool is_document_root() const noexcept; + simdjson_inline bool usable() const noexcept; + + /** The document this element references. */ + const dom::document *doc; + + /** The index of this element on `doc.tape[]` */ + size_t json_index; +}; + +} // namespace internal +} // namespace simdjson + +#endif // SIMDJSON_INTERNAL_TAPE_REF_H +/* end file simdjson/internal/tape_ref.h */ + +namespace simdjson { +namespace dom { + +/** + * JSON array. + */ +class array { +public: + /** Create a new, invalid array */ + simdjson_inline array() noexcept; + + class iterator { + public: + using value_type = element; + using difference_type = std::ptrdiff_t; + using pointer = void; + using reference = value_type; + using iterator_category = std::forward_iterator_tag; + + /** + * Get the actual value + */ + inline reference operator*() const noexcept; + /** + * Get the next value. + * + * Part of the std::iterator interface. + */ + inline iterator& operator++() noexcept; + /** + * Get the next value. + * + * Part of the std::iterator interface. + */ + inline iterator operator++(int) noexcept; + /** + * Check if these values come from the same place in the JSON. + * + * Part of the std::iterator interface. + */ + inline bool operator!=(const iterator& other) const noexcept; + inline bool operator==(const iterator& other) const noexcept; + + inline bool operator<(const iterator& other) const noexcept; + inline bool operator<=(const iterator& other) const noexcept; + inline bool operator>=(const iterator& other) const noexcept; + inline bool operator>(const iterator& other) const noexcept; + + iterator() noexcept = default; + iterator(const iterator&) noexcept = default; + iterator& operator=(const iterator&) noexcept = default; + private: + simdjson_inline iterator(const internal::tape_ref &tape) noexcept; + internal::tape_ref tape{}; + friend class array; + }; + + /** + * Return the first array element. + * + * Part of the std::iterable interface. + */ + inline iterator begin() const noexcept; + /** + * One past the last array element. + * + * Part of the std::iterable interface. + */ + inline iterator end() const noexcept; + /** + * Get the size of the array (number of immediate children). + * It is a saturated value with a maximum of 0xFFFFFF: if the value + * is 0xFFFFFF then the size is 0xFFFFFF or greater. + */ + inline size_t size() const noexcept; + /** + * Get the total number of slots used by this array on the tape. + * + * Note that this is not the same thing as `size()`, which reports the + * number of actual elements within an array (not counting its children). + * + * Since an element can use 1 or 2 slots on the tape, you can only use this + * to figure out the total size of an array (including its children, + * recursively) if you know its structure ahead of time. + **/ + inline size_t number_of_slots() const noexcept; + /** + * Get the value associated with the given JSON pointer. We use the RFC 6901 + * https://tools.ietf.org/html/rfc6901 standard, interpreting the current node + * as the root of its own JSON document. + * + * dom::parser parser; + * array a = parser.parse(R"([ { "foo": { "a": [ 10, 20, 30 ] }} ])"_padded); + * a.at_pointer("/0/foo/a/1") == 20 + * a.at_pointer("0")["foo"]["a"].at(1) == 20 + * + * @return The value associated with the given JSON pointer, or: + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed + */ + inline simdjson_result at_pointer(std::string_view json_pointer) const noexcept; + + /** + * Recursive function which processes the JSON path of each child element + */ + inline void process_json_path_of_child_elements(std::vector::iterator& current, std::vector::iterator& end, const std::string_view& path_suffix, std::vector& accumulator) const noexcept; + + + /** + * Adds support for JSONPath expression with wildcards '*' + */ + inline simdjson_result> at_path_with_wildcard(std::string_view json_path) const noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://www.rfc-editor.org/rfc/rfc9535 (RFC 9535) + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) const noexcept; + + /** + * Get the value at the given index. This function has linear-time complexity and + * is equivalent to the following: + * + * size_t i=0; + * for (auto element : *this) { + * if (i == index) { return element; } + * i++; + * } + * return INDEX_OUT_OF_BOUNDS; + * + * Avoid calling the at() function repeatedly. + * + * @return The value at the given index, or: + * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length + */ + inline simdjson_result at(size_t index) const noexcept; + + /** + * Gets the values of items in an array element + * This function has linear-time complexity: the values are checked one by one. + * + * @return The child elements of an array + */ + + inline std::vector& get_values(std::vector& out) const noexcept; + + /** + * Implicitly convert object to element + */ + inline operator element() const noexcept; + +private: + simdjson_inline array(const internal::tape_ref &tape) noexcept; + internal::tape_ref tape{}; + friend class element; + friend struct simdjson_result; + template + friend class simdjson::internal::string_builder; +}; + + +} // namespace dom + +/** The result of a JSON conversion that may fail. */ +template<> +struct simdjson_result : public internal::simdjson_result_base { +public: + simdjson_inline simdjson_result() noexcept; ///< @private + simdjson_inline simdjson_result(dom::array value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + + inline simdjson_result at_pointer(std::string_view json_pointer) const noexcept; + inline void process_json_path_of_child_elements(std::vector::iterator& current, std::vector::iterator& end, const std::string_view& path_suffix, std::vector& accumulator) const noexcept; + inline simdjson_result> at_path_with_wildcard(std::string_view json_path) const noexcept; + inline simdjson_result at_path(std::string_view json_path) const noexcept; + inline simdjson_result at(size_t index) const noexcept; + inline std::vector& get_values(std::vector& out) const noexcept; + +#if SIMDJSON_EXCEPTIONS + inline dom::array::iterator begin() const noexcept(false); + inline dom::array::iterator end() const noexcept(false); + inline size_t size() const noexcept(false); +#endif // SIMDJSON_EXCEPTIONS +}; + + + +} // namespace simdjson + +#if SIMDJSON_SUPPORTS_RANGES +namespace std { +namespace ranges { +template<> +inline constexpr bool enable_view = true; +#if SIMDJSON_EXCEPTIONS +template<> +inline constexpr bool enable_view> = true; +#endif // SIMDJSON_EXCEPTIONS +} // namespace ranges +} // namespace std +#endif // SIMDJSON_SUPPORTS_RANGES + +#endif // SIMDJSON_DOM_ARRAY_H +/* end file simdjson/dom/array.h */ +/* including simdjson/dom/document_stream.h: #include "simdjson/dom/document_stream.h" */ +/* begin file simdjson/dom/document_stream.h */ +#ifndef SIMDJSON_DOCUMENT_STREAM_H +#define SIMDJSON_DOCUMENT_STREAM_H + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* including simdjson/dom/parser.h: #include "simdjson/dom/parser.h" */ +/* begin file simdjson/dom/parser.h */ +#ifndef SIMDJSON_DOM_PARSER_H +#define SIMDJSON_DOM_PARSER_H + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* including simdjson/dom/document.h: #include "simdjson/dom/document.h" */ +/* begin file simdjson/dom/document.h */ +#ifndef SIMDJSON_DOM_DOCUMENT_H +#define SIMDJSON_DOM_DOCUMENT_H + +/* skipped duplicate #include "simdjson/dom/base.h" */ + +#include + +namespace simdjson { +namespace dom { + +/** + * A parsed JSON document. + * + * This class cannot be copied, only moved, to avoid unintended allocations. + */ +class document { +public: + /** + * Create a document container with zero capacity. + * + * The parser will allocate capacity as needed. + */ + document() noexcept = default; + ~document() noexcept = default; + + /** + * Take another document's buffers. + * + * @param other The document to take. Its capacity is zeroed and it is invalidated. + */ + document(document &&other) noexcept = default; + /** @private */ + document(const document &) = delete; // Disallow copying + /** + * Take another document's buffers. + * + * @param other The document to take. Its capacity is zeroed. + */ + document &operator=(document &&other) noexcept = default; + /** @private */ + document &operator=(const document &) = delete; // Disallow copying + + /** + * Get the root element of this document as a JSON array. + */ + element root() const noexcept; + + /** + * @private Dump the raw tape for debugging. + * + * @param os the stream to output to. + * @return false if the tape is likely wrong (e.g., you did not parse a valid JSON). + */ + bool dump_raw_tape(std::ostream &os) const noexcept; + + /** @private Structural values. */ + std::unique_ptr tape{}; + + /** @private String values. + * + * Should be at least byte_capacity. + */ + std::unique_ptr string_buf{}; + /** @private Allocate memory to support + * input JSON documents of up to len bytes. + * + * When calling this function, you lose + * all the data. + * + * The memory allocation is strict: you + * can you use this function to increase + * or lower the amount of allocated memory. + * Passing zero clears the memory. + */ + error_code allocate(size_t len) noexcept; + /** @private Capacity in bytes, in terms + * of how many bytes of input JSON we can + * support. + */ + size_t capacity() const noexcept; + + +private: + size_t allocated_capacity{0}; + friend class parser; +}; // class document + +} // namespace dom +} // namespace simdjson + +#endif // SIMDJSON_DOM_DOCUMENT_H +/* end file simdjson/dom/document.h */ + +namespace simdjson { + +namespace dom { + +/** + * A persistent document parser. + * + * The parser is designed to be reused, holding the internal buffers necessary to do parsing, + * as well as memory for a single document. The parsed document is overwritten on each parse. + * + * This class cannot be copied, only moved, to avoid unintended allocations. + * + * @note Moving a parser instance may invalidate "dom::element" instances. If you need to + * preserve both the "dom::element" instances and the parser, consider wrapping the parser + * instance in a std::unique_ptr instance: + * + * std::unique_ptr parser(new dom::parser{}); + * auto error = parser->load(f).get(root); + * + * You can then move std::unique_ptr safely. + * + * @note This is not thread safe: one parser cannot produce two documents at the same time! + */ +class parser { +public: + /** + * Create a JSON parser. + * + * The new parser will have zero capacity. + * + * @param max_capacity The maximum document length the parser can automatically handle. The parser + * will allocate more capacity on an as needed basis (when it sees documents too big to handle) + * up to this amount. The parser still starts with zero capacity no matter what this number is: + * to allocate an initial capacity, call allocate() after constructing the parser. + * Defaults to SIMDJSON_MAXSIZE_BYTES (the largest single document simdjson can process). + */ + simdjson_inline explicit parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES) noexcept; + /** + * Take another parser's buffers and state. + * + * @param other The parser to take. Its capacity is zeroed. + */ + simdjson_inline parser(parser &&other) noexcept; + parser(const parser &) = delete; ///< @private Disallow copying + /** + * Take another parser's buffers and state. + * + * @param other The parser to take. Its capacity is zeroed. + */ + simdjson_inline parser &operator=(parser &&other) noexcept; + parser &operator=(const parser &) = delete; ///< @private Disallow copying + + /** Deallocate the JSON parser. */ + ~parser()=default; + + /** + * Load a JSON document from a file and return a reference to it. + * + * dom::parser parser; + * const element doc = parser.load("jsonexamples/twitter.json"); + * + * The function is eager: the file's content is loaded in memory inside the parser instance + * and immediately parsed. The file can be deleted after the `parser.load` call. + * + * ### IMPORTANT: Document Lifetime + * + * The JSON document still lives in the parser: this is the most efficient way to parse JSON + * documents because it reuses the same buffers, but you *must* use the document before you + * destroy the parser or call parse() again. + * + * Moving the parser instance is safe, but it invalidates the element instances. You may store + * the parser instance without moving it by wrapping it inside an `unique_ptr` instance like + * so: `std::unique_ptr parser(new dom::parser{});`. + * + * ### Parser Capacity + * + * If the parser's current capacity is less than the file length, it will allocate enough capacity + * to handle it (up to max_capacity). + * + * ## Windows and Unicode + * + * Windows users who need to read files with non-ANSI characters in the + * name should set their code page to UTF-8 (65001) before calling this + * function. This should be the default with Windows 11 and better. + * Further, they may use the AreFileApisANSI function to determine whether + * the filename is interpreted using the ANSI or the system default OEM + * codepage, and they may call SetFileApisToOEM accordingly. + * + * @param path The path to load. + * @return The document, or an error: + * - IO_ERROR if there was an error opening or reading the file. + * Be mindful that on some 32-bit systems, + * the file size might be limited to 2 GB. + * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. + * - CAPACITY if the parser does not have enough capacity and len > max_capacity. + * - other json errors if parsing fails. You should not rely on these errors to always the same for the + * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). + */ + inline simdjson_result load(std::string_view path) & noexcept; + inline simdjson_result load(std::string_view path) && = delete ; + + /** + * Load a JSON document from a file into a provide document instance and return a temporary reference to it. + * It is similar to the function `load` except that instead of parsing into the internal + * `document` instance associated with the parser, it allows the user to provide a document + * instance. + * + * dom::parser parser; + * dom::document doc; + * element doc_root = parser.load_into_document(doc, "jsonexamples/twitter.json"); + * + * The function is eager: the file's content is loaded in memory inside the parser instance + * and immediately parsed. The file can be deleted after the `parser.load_into_document` call. + * + * ### IMPORTANT: Document Lifetime + * + * After the call to load_into_document, the parser is no longer needed. + * + * The JSON document lives in the document instance: you must keep the document + * instance alive while you navigate through it (i.e., used the returned value from + * load_into_document). You are encourage to reuse the document instance + * many times with new data to avoid reallocations: + * + * dom::document doc; + * element doc_root1 = parser.load_into_document(doc, "jsonexamples/twitter.json"); + * //... doc_root1 is a pointer inside doc + * element doc_root2 = parser.load_into_document(doc, "jsonexamples/twitter.json"); + * //... doc_root2 is a pointer inside doc + * // at this point doc_root1 is no longer safe + * + * Moving the document instance is safe, but it invalidates the element instances. After + * moving a document, you can recover safe access to the document root with its `root()` method. + * + * @param doc The document instance where the parsed data will be stored (on success). + * @param path The path to load. + * @return The document, or an error: + * - IO_ERROR if there was an error opening or reading the file. + * Be mindful that on some 32-bit systems, + * the file size might be limited to 2 GB. + * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. + * - CAPACITY if the parser does not have enough capacity and len > max_capacity. + * - other json errors if parsing fails. You should not rely on these errors to always the same for the + * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). + */ + inline simdjson_result load_into_document(document& doc, std::string_view path) & noexcept; + inline simdjson_result load_into_document(document& doc, std::string_view path) && =delete; + + /** + * Parse a JSON document and return a temporary reference to it. + * + * dom::parser parser; + * element doc_root = parser.parse(buf, len); + * + * The function eagerly parses the input: the input can be modified and discarded after + * the `parser.parse(buf, len)` call has completed. + * + * ### IMPORTANT: Document Lifetime + * + * The JSON document still lives in the parser: this is the most efficient way to parse JSON + * documents because it reuses the same buffers, but you *must* use the document before you + * destroy the parser or call parse() again. + * + * Moving the parser instance is safe, but it invalidates the element instances. You may store + * the parser instance without moving it by wrapping it inside an `unique_ptr` instance like + * so: `std::unique_ptr parser(new dom::parser{});`. + * + * ### REQUIRED: Buffer Padding + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you + * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the + * SIMDJSON_PADDING bytes to avoid runtime warnings. + * + * If realloc_if_needed is true (the default), it is assumed that the buffer does *not* have enough padding, + * and it is copied into an enlarged temporary buffer before parsing. Thus the following is safe: + * + * const char *json = R"({"key":"value"})"; + * const size_t json_len = std::strlen(json); + * simdjson::dom::parser parser; + * simdjson::dom::element element = parser.parse(json, json_len); + * + * If you set realloc_if_needed to false (e.g., parser.parse(json, json_len, false)), + * you must provide a buffer with at least SIMDJSON_PADDING extra bytes at the end. + * The benefit of setting realloc_if_needed to false is that you avoid a temporary + * memory allocation and a copy. + * + * The padded bytes may be read. It is not important how you initialize + * these bytes though we recommend a sensible default like null character values or spaces. + * For example, the following low-level code is safe: + * + * const char *json = R"({"key":"value"})"; + * const size_t json_len = std::strlen(json); + * std::unique_ptr padded_json_copy{new char[json_len + SIMDJSON_PADDING]}; + * std::memcpy(padded_json_copy.get(), json, json_len); + * std::memset(padded_json_copy.get() + json_len, '\0', SIMDJSON_PADDING); + * simdjson::dom::parser parser; + * simdjson::dom::element element = parser.parse(padded_json_copy.get(), json_len, false); + * + * ### std::string references + * + * Whenever you pass an std::string reference, the parser may access the bytes beyond the end of + * the string but before the end of the allocated memory (std::string::capacity()). + * If you are using a sanitizer that checks for reading uninitialized bytes or std::string's + * container-overflow checks, you may encounter sanitizer warnings. + * You can safely ignore these warnings. Or you can call simdjson::pad(std::string&) to pad the + * string with SIMDJSON_PADDING spaces: this function returns a simdjson::padding_string_view + * which can be be passed to the parser's parse function: + * + * std::string json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )"; + * element doc = parser.parse(simdjson::pad(json)); + * + * ### Parser Capacity + * + * If the parser's current capacity is less than len, it will allocate enough capacity + * to handle it (up to max_capacity). + * + * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless + * realloc_if_needed is true. + * @param len The length of the JSON. + * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. + * @return An element pointing at the root of the document, or an error: + * - MEMALLOC if realloc_if_needed is true or the parser does not have enough capacity, + * and memory allocation fails. + * - CAPACITY if the parser does not have enough capacity and len > max_capacity. + * - other json errors if parsing fails. You should not rely on these errors to always the same for the + * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). + */ + inline simdjson_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) & noexcept; + inline simdjson_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) && =delete; + /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */ + simdjson_inline simdjson_result parse(const char *buf, size_t len, bool realloc_if_needed = true) & noexcept; + simdjson_inline simdjson_result parse(const char *buf, size_t len, bool realloc_if_needed = true) && =delete; + /** @overload parse(const std::string &) */ + simdjson_inline simdjson_result parse(const std::string &s) & noexcept; + simdjson_inline simdjson_result parse(const std::string &s) && =delete; + /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */ + simdjson_inline simdjson_result parse(const padded_string &s) & noexcept; + simdjson_inline simdjson_result parse(const padded_string &s) && =delete; + /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */ + simdjson_inline simdjson_result parse(const padded_string_view &v) & noexcept; + simdjson_inline simdjson_result parse(const padded_string_view &v) && =delete; + + /** @private We do not want to allow implicit conversion from C string to std::string. */ + simdjson_inline simdjson_result parse(const char *buf) noexcept = delete; + + /** + * Parse a JSON document into a provide document instance and return a temporary reference to it. + * It is similar to the function `parse` except that instead of parsing into the internal + * `document` instance associated with the parser, it allows the user to provide a document + * instance. + * + * dom::parser parser; + * dom::document doc; + * element doc_root = parser.parse_into_document(doc, buf, len); + * + * The function eagerly parses the input: the input can be modified and discarded after + * the `parser.parse(buf, len)` call has completed. + * + * ### IMPORTANT: Document Lifetime + * + * After the call to parse_into_document, the parser is no longer needed. + * + * The JSON document lives in the document instance: you must keep the document + * instance alive while you navigate through it (i.e., used the returned value from + * parse_into_document). You are encourage to reuse the document instance + * many times with new data to avoid reallocations: + * + * dom::document doc; + * element doc_root1 = parser.parse_into_document(doc, buf1, len); + * //... doc_root1 is a pointer inside doc + * element doc_root2 = parser.parse_into_document(doc, buf1, len); + * //... doc_root2 is a pointer inside doc + * // at this point doc_root1 is no longer safe + * + * Moving the document instance is safe, but it invalidates the element instances. After + * moving a document, you can recover safe access to the document root with its `root()` method. + * + * @param doc The document instance where the parsed data will be stored (on success). + * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless + * realloc_if_needed is true. + * @param len The length of the JSON. + * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. + * @return An element pointing at the root of document, or an error: + * - MEMALLOC if realloc_if_needed is true or the parser does not have enough capacity, + * and memory allocation fails. + * - CAPACITY if the parser does not have enough capacity and len > max_capacity. + * - other json errors if parsing fails. You should not rely on these errors to always the same for the + * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). + */ + inline simdjson_result parse_into_document(document& doc, const uint8_t *buf, size_t len, bool realloc_if_needed = true) & noexcept; + inline simdjson_result parse_into_document(document& doc, const uint8_t *buf, size_t len, bool realloc_if_needed = true) && =delete; + /** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */ + simdjson_inline simdjson_result parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = true) & noexcept; + simdjson_inline simdjson_result parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = true) && =delete; + /** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */ + simdjson_inline simdjson_result parse_into_document(document& doc, const std::string &s) & noexcept; + simdjson_inline simdjson_result parse_into_document(document& doc, const std::string &s) && =delete; + /** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */ + simdjson_inline simdjson_result parse_into_document(document& doc, const padded_string &s) & noexcept; + simdjson_inline simdjson_result parse_into_document(document& doc, const padded_string &s) && =delete; + + /** @private We do not want to allow implicit conversion from C string to std::string. */ + simdjson_inline simdjson_result parse_into_document(document& doc, const char *buf) noexcept = delete; + + /** + * Load a file containing many JSON documents. + * + * dom::parser parser; + * for (const element doc : parser.load_many(path)) { + * cout << std::string(doc["title"]) << endl; + * } + * + * The file is loaded in memory and can be safely deleted after the `parser.load_many(path)` + * function has returned. The memory is held by the `parser` instance. + * + * The function is lazy: it may be that no more than one JSON document at a time is parsed. + * And, possibly, no document many have been parsed when the `parser.load_many(path)` function + * returned. + * + * If there is a UTF-8 BOM, the parser skips it. + * + * ### Format + * + * The file must contain a series of one or more JSON documents, concatenated into a single + * buffer, separated by whitespace. It effectively parses until it has a fully valid document, + * then starts parsing the next document at that point. (It does this with more parallelism and + * lookahead than you might think, though.) + * + * Documents that consist of an object or array may omit the whitespace between them, concatenating + * with no separator. documents that consist of a single primitive (i.e. documents that are not + * arrays or objects) MUST be separated with whitespace. + * + * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse. + * Setting batch_size to excessively large or excessively small values may impact negatively the + * performance. + * + * ### Error Handling + * + * All errors are returned during iteration: if there is a global error such as memory allocation, + * it will be yielded as the first result. Iteration always stops after the first error. + * + * As with all other simdjson methods, non-exception error handling is readily available through + * the same interface, requiring you to check the error before using the document: + * + * dom::parser parser; + * dom::document_stream docs; + * auto error = parser.load_many(path).get(docs); + * if (error) { cerr << error << endl; exit(1); } + * for (auto doc : docs) { + * std::string_view title; + * if ((error = doc["title"].get(title)) { cerr << error << endl; exit(1); } + * cout << title << endl; + * } + * + * ### Threads + * + * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the + * hood to do some lookahead. + * + * ### Parser Capacity + * + * If the parser's current capacity is less than batch_size, it will allocate enough capacity + * to handle it (up to max_capacity). + * + * @param path File name pointing at the concatenated JSON to parse. + * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet + * spot is cache-related: small enough to fit in cache, yet big enough to + * parse as many documents as possible in one tight loop. + * Defaults to 1MB (as simdjson::dom::DEFAULT_BATCH_SIZE), which has been a reasonable sweet + * spot in our tests. + * If you set the batch_size to a value smaller than simdjson::dom::MINIMAL_BATCH_SIZE + * (currently 32B), it will be replaced by simdjson::dom::MINIMAL_BATCH_SIZE. + * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: + * - IO_ERROR if there was an error opening or reading the file. + * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. + * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. + * - other json errors if parsing fails. You should not rely on these errors to always the same for the + * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). + */ + inline simdjson_result load_many(std::string_view path, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; + + /** + * Parse a buffer containing many JSON documents. + * + * dom::parser parser; + * for (element doc : parser.parse_many(buf, len)) { + * cout << std::string(doc["title"]) << endl; + * } + * + * No copy of the input buffer is made. + * + * The function is lazy: it may be that no more than one JSON document at a time is parsed. + * And, possibly, no document many have been parsed when the `parser.load_many(path)` function + * returned. + * + * The caller is responsabile to ensure that the input string data remains unchanged and is + * not deleted during the loop. In particular, the following is unsafe and will not compile: + * + * auto docs = parser.parse_many("[\"temporary data\"]"_padded); + * // here the string "[\"temporary data\"]" may no longer exist in memory + * // the parser instance may not have even accessed the input yet + * for (element doc : docs) { + * cout << std::string(doc["title"]) << endl; + * } + * + * The following is safe: + * + * auto json = "[\"temporary data\"]"_padded; + * auto docs = parser.parse_many(json); + * for (element doc : docs) { + * cout << std::string(doc["title"]) << endl; + * } + * + * If there is a UTF-8 BOM, the parser skips it. + * + * ### Format + * + * The buffer must contain a series of one or more JSON documents, concatenated into a single + * buffer, separated by whitespace. It effectively parses until it has a fully valid document, + * then starts parsing the next document at that point. (It does this with more parallelism and + * lookahead than you might think, though.) + * + * documents that consist of an object or array may omit the whitespace between them, concatenating + * with no separator. documents that consist of a single primitive (i.e. documents that are not + * arrays or objects) MUST be separated with whitespace. + * + * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse. + * Setting batch_size to excessively large or excessively small values may impact negatively the + * performance. + * + * ### Error Handling + * + * All errors are returned during iteration: if there is a global error such as memory allocation, + * it will be yielded as the first result. Iteration always stops after the first error. + * + * As with all other simdjson methods, non-exception error handling is readily available through + * the same interface, requiring you to check the error before using the document: + * + * dom::parser parser; + * dom::document_stream docs; + * auto error = parser.load_many(path).get(docs); + * if (error) { cerr << error << endl; exit(1); } + * for (auto doc : docs) { + * std::string_view title; + * if ((error = doc["title"].get(title)) { cerr << error << endl; exit(1); } + * cout << title << endl; + * } + * + * ### REQUIRED: Buffer Padding + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you + * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the + * SIMDJSON_PADDING bytes to avoid runtime warnings. + * + * ### Threads + * + * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the + * hood to do some lookahead. + * + * ### Parser Capacity + * + * If the parser's current capacity is less than batch_size, it will allocate enough capacity + * to handle it (up to max_capacity). + * + * @param buf The concatenated JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes. + * @param len The length of the concatenated JSON. + * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet + * spot is cache-related: small enough to fit in cache, yet big enough to + * parse as many documents as possible in one tight loop. + * Defaults to 10MB, which has been a reasonable sweet spot in our tests. + * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: + * - MEMALLOC if the parser does not have enough capacity and memory allocation fails + * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. + * - other json errors if parsing fails. You should not rely on these errors to always the same for the + * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). + */ + inline simdjson_result parse_many(const uint8_t *buf, size_t len, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result parse_many(const char *buf, size_t len, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result parse_many(const std::string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const std::string &&s, size_t batch_size) = delete;// unsafe + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result parse_many(const padded_string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; + inline simdjson_result parse_many(const padded_string &&s, size_t batch_size) = delete;// unsafe + + /** @private We do not want to allow implicit conversion from C string to std::string. */ + simdjson_result parse_many(const char *buf, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept = delete; + + /** + * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length + * and `max_depth` depth. + * + * @param capacity The new capacity. + * @param max_depth The new max_depth. Defaults to DEFAULT_MAX_DEPTH. + * @return The error, if there is one. + */ + simdjson_warn_unused inline error_code allocate(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept; + +#ifndef SIMDJSON_DISABLE_DEPRECATED_API + /** + * @private deprecated because it returns bool instead of error_code, which is our standard for + * failures. Use allocate() instead. + * + * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length + * and `max_depth` depth. + * + * @param capacity The new capacity. + * @param max_depth The new max_depth. Defaults to DEFAULT_MAX_DEPTH. + * @return true if successful, false if allocation failed. + */ + [[deprecated("Use allocate() instead.")]] + simdjson_warn_unused inline bool allocate_capacity(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept; +#endif // SIMDJSON_DISABLE_DEPRECATED_API + /** + * The largest document this parser can support without reallocating. + * + * @return Current capacity, in bytes. + */ + simdjson_inline size_t capacity() const noexcept; + + /** + * The largest document this parser can automatically support. + * + * The parser may reallocate internal buffers as needed up to this amount. + * + * @return Maximum capacity, in bytes. + */ + simdjson_inline size_t max_capacity() const noexcept; + + /** + * The maximum level of nested object and arrays supported by this parser. + * + * @return Maximum depth, in bytes. + */ + simdjson_pure simdjson_inline size_t max_depth() const noexcept; + + /** + * Set max_capacity. This is the largest document this parser can automatically support. + * + * The parser may reallocate internal buffers as needed up to this amount as documents are passed + * to it. + * + * Note: To avoid limiting the memory to an absurd value, such as zero or two bytes, + * iff you try to set max_capacity to a value lower than MINIMAL_DOCUMENT_CAPACITY, + * then the maximal capacity is set to MINIMAL_DOCUMENT_CAPACITY. + * + * This call will not allocate or deallocate, even if capacity is currently above max_capacity. + * + * @param max_capacity The new maximum capacity, in bytes. + */ + simdjson_inline void set_max_capacity(size_t max_capacity) noexcept; + +#ifdef SIMDJSON_THREADS_ENABLED + /** + * The parser instance can use threads when they are available to speed up some + * operations. It is enabled by default. Changing this attribute will change the + * behavior of the parser for future operations. Set to true by default. + */ + bool threaded{true}; +#else + /** + * When SIMDJSON_THREADS_ENABLED is not defined, the parser instance cannot use threads. + */ + bool threaded{false}; +#endif + /** @private Use the new DOM API instead */ + class Iterator; + /** @private Use simdjson_error instead */ + using InvalidJSON [[deprecated("Use simdjson_error instead")]] = simdjson_error; + + /** @private [for benchmarking access] The implementation to use */ + std::unique_ptr implementation{}; + + /** @private Use `if (parser.parse(...).error())` instead */ + bool valid{false}; + /** @private Use `parser.parse(...).error()` instead */ + error_code error{UNINITIALIZED}; + + /** @private Use `parser.parse(...).value()` instead */ + document doc{}; + + /** @private returns true if the document parsed was valid */ + [[deprecated("Use the result of parser.parse() instead")]] + inline bool is_valid() const noexcept; + + /** + * @private return an error code corresponding to the last parsing attempt, see + * simdjson.h will return UNINITIALIZED if no parsing was attempted + */ + [[deprecated("Use the result of parser.parse() instead")]] + inline int get_error_code() const noexcept; + + /** @private return the string equivalent of "get_error_code" */ + [[deprecated("Use error_message() on the result of parser.parse() instead, or cout << error")]] + inline std::string get_error_message() const noexcept; + + /** @private */ + [[deprecated("Use cout << on the result of parser.parse() instead")]] + inline bool print_json(std::ostream &os) const noexcept; + + /** @private Private and deprecated: use `parser.parse(...).doc.dump_raw_tape()` instead */ + inline bool dump_raw_tape(std::ostream &os) const noexcept; + + + /** + * When enabled, big integers (exceeding uint64 range) are stored as strings + * in the tape instead of returning BIGINT_ERROR. Default: false. + */ + inline void number_as_string(bool enabled) noexcept { _number_as_string = enabled; } + inline bool number_as_string() const noexcept { return _number_as_string; } + +private: + /** + * The maximum document length this parser will automatically support. + * + * The parser will not be automatically allocated above this amount. + */ + size_t _max_capacity; + + /** Whether to store big integers as strings instead of returning BIGINT_ERROR */ + bool _number_as_string{false}; + + /** + * The loaded buffer (reused each time load() is called) + */ + std::unique_ptr loaded_bytes; + + /** Capacity of loaded_bytes buffer. */ + size_t _loaded_bytes_capacity{0}; + + // all nodes are stored on the doc.tape using a 64-bit word. + // + // strings, double and ints are stored as + // a 64-bit word with a pointer to the actual value + // + // + // + // for objects or arrays, store [ or { at the beginning and } and ] at the + // end. For the openings ([ or {), we annotate them with a reference to the + // location on the doc.tape of the end, and for then closings (} and ]), we + // annotate them with a reference to the location of the opening + // + // + + /** + * Ensure we have enough capacity to handle at least desired_capacity bytes, + * and auto-allocate if not. This also allocates memory if needed in the + * internal document. + */ + inline error_code ensure_capacity(size_t desired_capacity) noexcept; + /** + * Ensure we have enough capacity to handle at least desired_capacity bytes, + * and auto-allocate if not. This also allocates memory if needed in the + * provided document. + */ + inline error_code ensure_capacity(document& doc, size_t desired_capacity) noexcept; + + /** Read the file into loaded_bytes */ + inline simdjson_result read_file(std::string_view path) noexcept; + + friend class parser::Iterator; + friend class document_stream; + + +}; // class parser + +} // namespace dom +} // namespace simdjson + +#endif // SIMDJSON_DOM_PARSER_H +/* end file simdjson/dom/parser.h */ + +#ifdef SIMDJSON_THREADS_ENABLED +#include +#include +#include +#endif + +namespace simdjson { +namespace dom { + +#ifdef SIMDJSON_THREADS_ENABLED +/** @private Custom worker class **/ +struct stage1_worker { + stage1_worker() noexcept = default; + stage1_worker(const stage1_worker&) = delete; + stage1_worker(stage1_worker&&) = delete; + stage1_worker operator=(const stage1_worker&) = delete; + ~stage1_worker(); + /** + * We only start the thread when it is needed, not at object construction, this may throw. + * You should only call this once. + **/ + void start_thread(); + /** + * Start a stage 1 job. You should first call 'run', then 'finish'. + * You must call start_thread once before. + */ + void run(document_stream * ds, dom::parser * stage1, size_t next_batch_start); + /** Wait for the run to finish (blocking). You should first call 'run', then 'finish'. **/ + void finish(); + +private: + + /** + * Normally, we would never stop the thread. But we do in the destructor. + * This function is only safe assuming that you are not waiting for results. You + * should have called run, then finish, and be done. + **/ + void stop_thread(); + + std::thread thread{}; + /** These three variables define the work done by the thread. **/ + dom::parser * stage1_thread_parser{}; + size_t _next_batch_start{}; + document_stream * owner{}; + /** + * We have two state variables. This could be streamlined to one variable in the future but + * we use two for clarity. + */ + bool has_work{false}; + bool can_work{true}; + + /** + * We lock using a mutex. + */ + std::mutex locking_mutex{}; + std::condition_variable cond_var{}; +}; +#endif + +/** + * A forward-only stream of documents. + * + * Produced by parser::parse_many. + * + */ +class document_stream { +public: + /** + * Construct an uninitialized document_stream. + * + * ```cpp + * document_stream docs; + * error = parser.parse_many(json).get(docs); + * ``` + */ + simdjson_inline document_stream() noexcept; + /** Move one document_stream to another. */ + simdjson_inline document_stream(document_stream &&other) noexcept = default; + /** Move one document_stream to another. */ + simdjson_inline document_stream &operator=(document_stream &&other) noexcept = default; + + simdjson_inline ~document_stream() noexcept; + /** + * Returns the input size in bytes. + */ + inline size_t size_in_bytes() const noexcept; + /** + * After iterating through the stream, this method + * returns the number of bytes that were not parsed at the end + * of the stream. If truncated_bytes() differs from zero, + * then the input was truncated maybe because incomplete JSON + * documents were found at the end of the stream. You + * may need to process the bytes in the interval [size_in_bytes()-truncated_bytes(), size_in_bytes()). + * + * You should only call truncated_bytes() after streaming through all + * documents, like so: + * + * document_stream stream = parser.parse_many(json,window); + * for(auto doc : stream) { + * // do something with doc + * } + * size_t truncated = stream.truncated_bytes(); + * + */ + inline size_t truncated_bytes() const noexcept; + /** + * An iterator through a forward-only stream of documents. + */ + class iterator { + public: + using value_type = simdjson_result; + using reference = value_type; + + using difference_type = std::ptrdiff_t; + + using iterator_category = std::input_iterator_tag; + + /** + * Default constructor. + */ + simdjson_inline iterator() noexcept; + /** + * Get the current document (or error). + */ + simdjson_inline reference operator*() noexcept; + /** + * Advance to the next document (prefix). + */ + inline iterator& operator++() noexcept; + /** + * Check if we're at the end yet. + * @param other the end iterator to compare to. + */ + simdjson_inline bool operator!=(const iterator &other) const noexcept; + /** + * @private + * + * Gives the current index in the input document in bytes. + * + * document_stream stream = parser.parse_many(json,window); + * for(auto i = stream.begin(); i != stream.end(); ++i) { + * auto doc = *i; + * size_t index = i.current_index(); + * } + * + * This function (current_index()) is experimental and the usage + * may change in future versions of simdjson: we find the API somewhat + * awkward and we would like to offer something friendlier. + */ + simdjson_inline size_t current_index() const noexcept; + /** + * @private + * + * Gives a view of the current document. + * + * document_stream stream = parser.parse_many(json,window); + * for(auto i = stream.begin(); i != stream.end(); ++i) { + * auto doc = *i; + * std::string_view v = i->source(); + * } + * + * The returned string_view instance is simply a map to the (unparsed) + * source string: it may thus include white-space characters and all manner + * of padding. + * + * This function (source()) is experimental and the usage + * may change in future versions of simdjson: we find the API somewhat + * awkward and we would like to offer something friendlier. + */ + simdjson_inline std::string_view source() const noexcept; + + private: + simdjson_inline iterator(document_stream *s, bool finished) noexcept; + /** The document_stream we're iterating through. */ + document_stream* stream; + /** Whether we're finished or not. */ + bool finished; + friend class document_stream; + }; + + /** + * Start iterating the documents in the stream. + */ + simdjson_inline iterator begin() noexcept; + /** + * The end of the stream, for iterator comparison purposes. + */ + simdjson_inline iterator end() noexcept; + +private: + + document_stream &operator=(const document_stream &) = delete; // Disallow copying + document_stream(const document_stream &other) = delete; // Disallow copying + + /** + * Construct a document_stream. Does not allocate or parse anything until the iterator is + * used. + * + * @param parser is a reference to the parser instance used to generate this document_stream + * @param buf is the raw byte buffer we need to process + * @param len is the length of the raw byte buffer in bytes + * @param batch_size is the size of the windows (must be strictly greater or equal to the largest JSON document) + */ + simdjson_inline document_stream( + dom::parser &parser, + const uint8_t *buf, + size_t len, + size_t batch_size + ) noexcept; + + /** + * Parse the first document in the buffer. Used by begin(), to handle allocation and + * initialization. + */ + inline void start() noexcept; + + /** + * Parse the next document found in the buffer previously given to document_stream. + * + * The content should be a valid JSON document encoded as UTF-8. If there is a + * UTF-8 BOM, the parser skips it. + * + * You do NOT need to pre-allocate a parser. This function takes care of + * pre-allocating a capacity defined by the batch_size defined when creating the + * document_stream object. + * + * The function returns simdjson::EMPTY if there is no more data to be parsed. + * + * The function returns simdjson::SUCCESS (as integer = 0) in case of success + * and indicates that the buffer has successfully been parsed to the end. + * Every document it contained has been parsed without error. + * + * The function returns an error code from simdjson/simdjson.h in case of failure + * such as simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth; + * the simdjson::error_message function converts these error codes into a string). + * + * You can also check validity by calling parser.is_valid(). The same parser can + * and should be reused for the other documents in the buffer. + */ + inline void next() noexcept; + + /** + * Pass the next batch through stage 1 and return when finished. + * When threads are enabled, this may wait for the stage 1 thread to finish. + */ + inline void load_batch() noexcept; + + /** Get the next document index. */ + inline size_t next_batch_start() const noexcept; + + /** Pass the next batch through stage 1 with the given parser. */ + inline error_code run_stage1(dom::parser &p, size_t batch_start) noexcept; + + dom::parser *parser; + const uint8_t *buf; + size_t len; + size_t batch_size; + /** The error (or lack thereof) from the current document. */ + error_code error; + size_t batch_start{0}; + size_t doc_index{}; +#ifdef SIMDJSON_THREADS_ENABLED + /** Indicates whether we use threads. Note that this needs to be a constant during the execution of the parsing. */ + bool use_thread; + + inline void load_from_stage1_thread() noexcept; + + /** Start a thread to run stage 1 on the next batch. */ + inline void start_stage1_thread() noexcept; + + /** Wait for the stage 1 thread to finish and capture the results. */ + inline void finish_stage1_thread() noexcept; + + /** The error returned from the stage 1 thread. */ + error_code stage1_thread_error{UNINITIALIZED}; + /** The thread used to run stage 1 against the next batch in the background. */ + friend struct stage1_worker; + std::unique_ptr worker{new(std::nothrow) stage1_worker()}; + /** + * The parser used to run stage 1 in the background. Will be swapped + * with the regular parser when finished. + */ + dom::parser stage1_thread_parser{}; +#endif // SIMDJSON_THREADS_ENABLED + + friend class dom::parser; + friend struct simdjson_result; + friend struct internal::simdjson_result_base; + +}; // class document_stream + +} // namespace dom + +template<> +struct simdjson_result : public internal::simdjson_result_base { +public: + simdjson_inline simdjson_result() noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result(dom::document_stream &&value) noexcept; ///< @private + +#if SIMDJSON_EXCEPTIONS + simdjson_inline dom::document_stream::iterator begin() noexcept(false); + simdjson_inline dom::document_stream::iterator end() noexcept(false); +#else // SIMDJSON_EXCEPTIONS +#ifndef SIMDJSON_DISABLE_DEPRECATED_API + [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] + simdjson_inline dom::document_stream::iterator begin() noexcept; + [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] + simdjson_inline dom::document_stream::iterator end() noexcept; +#endif // SIMDJSON_DISABLE_DEPRECATED_API +#endif // SIMDJSON_EXCEPTIONS +}; // struct simdjson_result + +} // namespace simdjson + +#endif // SIMDJSON_DOCUMENT_STREAM_H +/* end file simdjson/dom/document_stream.h */ +/* skipped duplicate #include "simdjson/dom/document.h" */ +/* including simdjson/dom/element.h: #include "simdjson/dom/element.h" */ +/* begin file simdjson/dom/element.h */ +#ifndef SIMDJSON_DOM_ELEMENT_H +#define SIMDJSON_DOM_ELEMENT_H + +#include + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* skipped duplicate #include "simdjson/dom/array.h" */ + +namespace simdjson { +namespace dom { + +/** + * The actual concrete type of a JSON element + * This is the type it is most easily cast to with get<>. + */ +enum class element_type { + ARRAY = '[', ///< dom::array + OBJECT = '{', ///< dom::object + INT64 = 'l', ///< int64_t + UINT64 = 'u', ///< uint64_t: any integer that fits in uint64_t but *not* int64_t + DOUBLE = 'd', ///< double: Any number with a "." or "e" that fits in double. + STRING = '"', ///< std::string_view + BOOL = 't', ///< bool + NULL_VALUE = 'n', ///< null + BIGINT = 'Z' ///< std::string_view: big integer stored as raw digit string +}; + +/** + * A JSON element. + * + * References an element in a JSON document, representing a JSON null, boolean, string, number, + * array or object. + */ +class element { +public: + /** Create a new, invalid element. */ + simdjson_inline element() noexcept; + + /** The type of this element. */ + simdjson_inline element_type type() const noexcept; + + /** + * Cast this element to an array. + * + * @returns An object that can be used to iterate the array, or: + * INCORRECT_TYPE if the JSON element is not an array. + */ + inline simdjson_result get_array() const noexcept; + /** + * Cast this element to an object. + * + * @returns An object that can be used to look up or iterate the object's fields, or: + * INCORRECT_TYPE if the JSON element is not an object. + */ + inline simdjson_result get_object() const noexcept; + /** + * Cast this element to a null-terminated C string. + * + * The string is guaranteed to be valid UTF-8. + * + * The length of the string is given by get_string_length(). Because JSON strings + * may contain null characters, it may be incorrect to use strlen to determine the + * string length. + * + * It is possible to get a single string_view instance which represents both the string + * content and its length: see get_string(). + * + * @returns A pointer to a null-terminated UTF-8 string. This string is stored in the parser and will + * be invalidated the next time it parses a document or when it is destroyed. + * Returns INCORRECT_TYPE if the JSON element is not a string. + */ + inline simdjson_result get_c_str() const noexcept; + /** + * Gives the length in bytes of the string. + * + * It is possible to get a single string_view instance which represents both the string + * content and its length: see get_string(). + * + * @returns A string length in bytes. + * Returns INCORRECT_TYPE if the JSON element is not a string. + */ + inline simdjson_result get_string_length() const noexcept; + /** + * Cast this element to a string. + * + * The string is guaranteed to be valid UTF-8. + * + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next time it + * parses a document or when it is destroyed. + * Returns INCORRECT_TYPE if the JSON element is not a string. + */ + inline simdjson_result get_string() const noexcept; + /** + * Cast this element to a signed integer. + * + * @returns A signed 64-bit integer. + * Returns INCORRECT_TYPE if the JSON element is not an integer, or NUMBER_OUT_OF_RANGE + * if it is negative. + */ + inline simdjson_result get_int64() const noexcept; + /** + * Cast this element to an unsigned integer. + * + * @returns An unsigned 64-bit integer. + * Returns INCORRECT_TYPE if the JSON element is not an integer, or NUMBER_OUT_OF_RANGE + * if it is too large. + */ + inline simdjson_result get_uint64() const noexcept; + /** + * Cast this element to a double floating-point. + * + * @returns A double value. + * Returns INCORRECT_TYPE if the JSON element is not a number. + */ + inline simdjson_result get_double() const noexcept; + /** + * Cast this element to a bool. + * + * @returns A bool value. + * Returns INCORRECT_TYPE if the JSON element is not a boolean. + */ + inline simdjson_result get_bool() const noexcept; + + /** + * Read this element as a big integer (raw digit string). + * + * @returns A string_view of the raw digits, or: + * INCORRECT_TYPE if the JSON element is not a big integer. + */ + inline simdjson_result get_bigint() const noexcept; + + /** + * Whether this element is a json array. + * + * Equivalent to is(). + */ + inline bool is_array() const noexcept; + /** + * Whether this element is a json object. + * + * Equivalent to is(). + */ + inline bool is_object() const noexcept; + /** + * Whether this element is a json string. + * + * Equivalent to is() or is(). + */ + inline bool is_string() const noexcept; + /** + * Whether this element is a json number that fits in a signed 64-bit integer. + * + * Equivalent to is(). + */ + inline bool is_int64() const noexcept; + /** + * Whether this element is a json number that fits in an unsigned 64-bit integer. + * + * Equivalent to is(). + */ + inline bool is_uint64() const noexcept; + /** + * Whether this element is a json number that fits in a double. + * + * Equivalent to is(). + */ + inline bool is_double() const noexcept; + + /** + * Whether this element is a json number. + * + * Both integers and floating points will return true. + */ + inline bool is_number() const noexcept; + + /** + * Whether this element is a json `true` or `false`. + * + * Equivalent to is(). + */ + inline bool is_bool() const noexcept; + /** + * Whether this element is a json `null`. + */ + inline bool is_null() const noexcept; + + /** + * Whether this element is a big integer (number exceeding 64-bit range). + */ + inline bool is_bigint() const noexcept; + + /** + * Tell whether the value can be cast to provided type (T). + * + * Supported types: + * - Boolean: bool + * - Number: double, uint64_t, int64_t + * - String: std::string_view, const char * + * - Array: dom::array + * - Object: dom::object + * + * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object + */ + template + simdjson_inline bool is() const noexcept; + + /** + * Get the value as the provided type (T). + * + * Supported types: + * - Boolean: bool + * - Number: double, uint64_t, int64_t + * - String: std::string_view, const char * + * - Array: dom::array + * - Object: dom::object + * + * You may use get_double(), get_bool(), get_uint64(), get_int64(), + * get_object(), get_array() or get_string() instead. + * + * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object + * + * @returns The value cast to the given type, or: + * INCORRECT_TYPE if the value cannot be cast to the given type. + */ + + template + inline simdjson_result get() const noexcept { + // Unless the simdjson library provides an inline implementation, calling this method should + // immediately fail. + static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " + "The supported types are Boolean (bool), numbers (double, uint64_t, int64_t), " + "strings (std::string_view, const char *), arrays (dom::array) and objects (dom::object). " + "We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " + "get_object(), get_array() or get_string() instead of the get template."); + } + + /** + * Get the value as the provided type (T). + * + * Supported types: + * - Boolean: bool + * - Number: double, uint64_t, int64_t + * - String: std::string_view, const char * + * - Array: dom::array + * - Object: dom::object + * + * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object + * + * @param value The variable to set to the value. May not be set if there is an error. + * + * @returns The error that occurred, or SUCCESS if there was no error. + */ + template + simdjson_warn_unused simdjson_inline error_code get(T &value) const noexcept; + + /** + * Get the value as the provided type (T), setting error if it's not the given type. + * + * Supported types: + * - Boolean: bool + * - Number: double, uint64_t, int64_t + * - String: std::string_view, const char * + * - Array: dom::array + * - Object: dom::object + * + * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object + * + * @param value The variable to set to the given type. value is undefined if there is an error. + * @param error The variable to store the error. error is set to error_code::SUCCEED if there is an error. + */ + template + inline void tie(T &value, error_code &error) && noexcept; + +#if SIMDJSON_EXCEPTIONS + /** + * Read this element as a boolean. + * + * @return The boolean value + * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not a boolean. + */ + inline operator bool() const noexcept(false); + + /** + * Read this element as a null-terminated UTF-8 string. + * + * Be mindful that JSON allows strings to contain null characters. + * + * Does *not* convert other types to a string; requires that the JSON type of the element was + * an actual string. + * + * @return The string value. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not a string. + */ + inline explicit operator const char*() const noexcept(false); + + /** + * Read this element as a null-terminated UTF-8 string. + * + * Does *not* convert other types to a string; requires that the JSON type of the element was + * an actual string. + * + * @return The string value. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not a string. + */ + inline operator std::string_view() const noexcept(false); + + /** + * Read this element as an unsigned integer. + * + * @return The integer value. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an integer + * @exception simdjson_error(NUMBER_OUT_OF_RANGE) if the integer does not fit in 64 bits or is negative + */ + inline operator uint64_t() const noexcept(false); + /** + * Read this element as an signed integer. + * + * @return The integer value. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an integer + * @exception simdjson_error(NUMBER_OUT_OF_RANGE) if the integer does not fit in 64 bits + */ + inline operator int64_t() const noexcept(false); + /** + * Read this element as an double. + * + * @return The double value. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not a number + */ + inline operator double() const noexcept(false); + /** + * Read this element as a JSON array. + * + * @return The JSON array. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an array + */ + inline operator array() const noexcept(false); + /** + * Read this element as a JSON object (key/value pairs). + * + * @return The JSON object. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an object + */ + inline operator object() const noexcept(false); + + /** + * Iterate over each element in this array. + * + * @return The beginning of the iteration. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an array + */ + inline dom::array::iterator begin() const noexcept(false); + + /** + * Iterate over each element in this array. + * + * @return The end of the iteration. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an array + */ + inline dom::array::iterator end() const noexcept(false); +#endif // SIMDJSON_EXCEPTIONS + + /** + * Get the value associated with the given key. + * + * The key will be matched against **unescaped** JSON: + * + * dom::parser parser; + * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + * - INCORRECT_TYPE if this is not an object + */ + inline simdjson_result operator[](std::string_view key) const noexcept; + + /** + * Get the value associated with the given key. + * + * The key will be matched against **unescaped** JSON: + * + * dom::parser parser; + * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + * - INCORRECT_TYPE if this is not an object + */ + inline simdjson_result operator[](const char *key) const noexcept; + simdjson_result operator[](int) const noexcept = delete; + + + /** + * Get the value associated with the given JSON pointer. We use the RFC 6901 + * https://tools.ietf.org/html/rfc6901 standard. + * + * dom::parser parser; + * element doc = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded); + * doc.at_pointer("/foo/a/1") == 20 + * doc.at_pointer("/foo")["a"].at(1) == 20 + * doc.at_pointer("")["foo"]["a"].at(1) == 20 + * + * It is allowed for a key to be the empty string: + * + * dom::parser parser; + * object obj = parser.parse(R"({ "": { "a": [ 10, 20, 30 ] }})"_padded); + * obj.at_pointer("//a/1") == 20 + * + * @return The value associated with the given JSON pointer, or: + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed + */ + inline simdjson_result at_pointer(const std::string_view json_pointer) const noexcept; + + inline simdjson_result> at_path_with_wildcard(const std::string_view json_path) const noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://www.rfc-editor.org/rfc/rfc9535 (RFC 9535) + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) const noexcept; + +#ifndef SIMDJSON_DISABLE_DEPRECATED_API + /** + * + * Version 0.4 of simdjson used an incorrect interpretation of the JSON Pointer standard + * and allowed the following : + * + * dom::parser parser; + * element doc = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded); + * doc.at("foo/a/1") == 20 + * + * Though it is intuitive, it is not compliant with RFC 6901 + * https://tools.ietf.org/html/rfc6901 + * + * For standard compliance, use the at_pointer function instead. + * + * @return The value associated with the given JSON pointer, or: + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed + */ + [[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] + inline simdjson_result at(const std::string_view json_pointer) const noexcept; +#endif // SIMDJSON_DISABLE_DEPRECATED_API + + /** + * Get the value at the given index. + * + * @return The value at the given index, or: + * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length + */ + inline simdjson_result at(size_t index) const noexcept; + + /** + * Get the value associated with the given key. + * + * The key will be matched against **unescaped** JSON: + * + * dom::parser parser; + * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + */ + inline simdjson_result at_key(std::string_view key) const noexcept; + + /** + * Get the value associated with the given key in a case-insensitive manner. + * + * Note: The key will be matched against **unescaped** JSON. + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + */ + inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; + + /** + * operator< defines a total order for element allowing to use them in + * ordered C++ STL containers + * + * @return TRUE if the key appears before the other one in the tape + */ + inline bool operator<(const element &other) const noexcept; + + /** + * operator== allows to verify if two element values reference the + * same JSON item + * + * @return TRUE if the two values references the same JSON element + */ + inline bool operator==(const element &other) const noexcept; + + /** @private for debugging. Prints out the root element. */ + inline bool dump_raw_tape(std::ostream &out) const noexcept; + +private: + simdjson_inline element(const internal::tape_ref &tape) noexcept; + internal::tape_ref tape{}; + friend class document; + friend class object; + friend class array; + friend struct simdjson_result; + template + friend class simdjson::internal::string_builder; + +}; + +} // namespace dom + +/** The result of a JSON navigation that may fail. */ +template<> +struct simdjson_result : public internal::simdjson_result_base { +public: + simdjson_inline simdjson_result() noexcept; ///< @private + simdjson_inline simdjson_result(dom::element &&value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + + simdjson_inline simdjson_result type() const noexcept; + template + simdjson_inline bool is() const noexcept; + template + simdjson_inline simdjson_result get() const noexcept; + template + simdjson_warn_unused simdjson_inline error_code get(T &value) const noexcept; + + simdjson_inline simdjson_result get_array() const noexcept; + simdjson_inline simdjson_result get_object() const noexcept; + simdjson_inline simdjson_result get_c_str() const noexcept; + simdjson_inline simdjson_result get_string_length() const noexcept; + simdjson_inline simdjson_result get_string() const noexcept; + simdjson_inline simdjson_result get_int64() const noexcept; + simdjson_inline simdjson_result get_uint64() const noexcept; + simdjson_inline simdjson_result get_double() const noexcept; + simdjson_inline simdjson_result get_bool() const noexcept; + simdjson_inline simdjson_result get_bigint() const noexcept; + + simdjson_inline bool is_array() const noexcept; + simdjson_inline bool is_object() const noexcept; + simdjson_inline bool is_string() const noexcept; + simdjson_inline bool is_int64() const noexcept; + simdjson_inline bool is_uint64() const noexcept; + simdjson_inline bool is_double() const noexcept; + simdjson_inline bool is_number() const noexcept; + simdjson_inline bool is_bool() const noexcept; + simdjson_inline bool is_null() const noexcept; + simdjson_inline bool is_bigint() const noexcept; + + simdjson_inline simdjson_result operator[](std::string_view key) const noexcept; + simdjson_inline simdjson_result operator[](const char *key) const noexcept; + simdjson_result operator[](int) const noexcept = delete; + simdjson_inline simdjson_result at_pointer(const std::string_view json_pointer) const noexcept; + simdjson_inline simdjson_result> at_path_with_wildcard(const std::string_view json_path) const noexcept; + simdjson_inline simdjson_result at_path(const std::string_view json_path) const noexcept; + [[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] + simdjson_inline simdjson_result at(const std::string_view json_pointer) const noexcept; + simdjson_inline simdjson_result at(size_t index) const noexcept; + simdjson_inline simdjson_result at_key(std::string_view key) const noexcept; + simdjson_inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; + +#if SIMDJSON_EXCEPTIONS + simdjson_inline operator bool() const noexcept(false); + simdjson_inline explicit operator const char*() const noexcept(false); + simdjson_inline operator std::string_view() const noexcept(false); + simdjson_inline operator uint64_t() const noexcept(false); + simdjson_inline operator int64_t() const noexcept(false); + simdjson_inline operator double() const noexcept(false); + simdjson_inline operator dom::array() const noexcept(false); + simdjson_inline operator dom::object() const noexcept(false); + + simdjson_inline dom::array::iterator begin() const noexcept(false); + simdjson_inline dom::array::iterator end() const noexcept(false); +#endif // SIMDJSON_EXCEPTIONS +}; + +} // namespace simdjson + +#endif // SIMDJSON_DOM_DOCUMENT_H +/* end file simdjson/dom/element.h */ +/* including simdjson/dom/object.h: #include "simdjson/dom/object.h" */ +/* begin file simdjson/dom/object.h */ +#ifndef SIMDJSON_DOM_OBJECT_H +#define SIMDJSON_DOM_OBJECT_H + +#include + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* skipped duplicate #include "simdjson/dom/element.h" */ +/* skipped duplicate #include "simdjson/internal/tape_ref.h" */ + +namespace simdjson { +namespace dom { + +/** + * JSON object. + */ +class object { +public: + /** Create a new, invalid object */ + simdjson_inline object() noexcept; + + class iterator { + public: + using value_type = const key_value_pair; + using difference_type = std::ptrdiff_t; + using pointer = void; + using reference = value_type; + using iterator_category = std::forward_iterator_tag; + + /** + * Get the actual key/value pair + */ + inline reference operator*() const noexcept; + /** + * Get the next key/value pair. + * + * Part of the std::iterator interface. + * + */ + inline iterator& operator++() noexcept; + /** + * Get the next key/value pair. + * + * Part of the std::iterator interface. + * + */ + inline iterator operator++(int) noexcept; + /** + * Check if these values come from the same place in the JSON. + * + * Part of the std::iterator interface. + */ + inline bool operator!=(const iterator& other) const noexcept; + inline bool operator==(const iterator& other) const noexcept; + + inline bool operator<(const iterator& other) const noexcept; + inline bool operator<=(const iterator& other) const noexcept; + inline bool operator>=(const iterator& other) const noexcept; + inline bool operator>(const iterator& other) const noexcept; + /** + * Get the key of this key/value pair. + */ + inline std::string_view key() const noexcept; + /** + * Get the length (in bytes) of the key in this key/value pair. + * You should expect this function to be faster than key().size(). + */ + inline uint32_t key_length() const noexcept; + /** + * Returns true if the key in this key/value pair is equal + * to the provided string_view. + */ + inline bool key_equals(std::string_view o) const noexcept; + /** + * Returns true if the key in this key/value pair is equal + * to the provided string_view in a case-insensitive manner. + * Case comparisons may only be handled correctly for ASCII strings. + */ + inline bool key_equals_case_insensitive(std::string_view o) const noexcept; + /** + * Get the key of this key/value pair. + */ + inline const char *key_c_str() const noexcept; + /** + * Get the value of this key/value pair. + */ + inline element value() const noexcept; + + iterator() noexcept = default; + iterator(const iterator&) noexcept = default; + iterator& operator=(const iterator&) noexcept = default; + private: + simdjson_inline iterator(const internal::tape_ref &tape) noexcept; + + internal::tape_ref tape{}; + + friend class object; + }; + + /** + * Return the first key/value pair. + * + * Part of the std::iterable interface. + */ + inline iterator begin() const noexcept; + /** + * One past the last key/value pair. + * + * Part of the std::iterable interface. + */ + inline iterator end() const noexcept; + /** + * Get the size of the object (number of keys). + * It is a saturated value with a maximum of 0xFFFFFF: if the value + * is 0xFFFFFF then the size is 0xFFFFFF or greater. + */ + inline size_t size() const noexcept; + /** + * Get the value associated with the given key. + * + * The key will be matched against **unescaped** JSON: + * + * dom::parser parser; + * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD + * + * This function has linear-time complexity: the keys are checked one by one. + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + * - INCORRECT_TYPE if this is not an object + */ + inline simdjson_result operator[](std::string_view key) const noexcept; + + /** + * Get the value associated with the given key. + * + * The key will be matched against **unescaped** JSON: + * + * dom::parser parser; + * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD + * + * This function has linear-time complexity: the keys are checked one by one. + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + * - INCORRECT_TYPE if this is not an object + */ + inline simdjson_result operator[](const char *key) const noexcept; + simdjson_result operator[](int) const noexcept = delete; + + /** + * Get the value associated with the given JSON pointer. We use the RFC 6901 + * https://tools.ietf.org/html/rfc6901 standard, interpreting the current node + * as the root of its own JSON document. + * + * dom::parser parser; + * object obj = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded); + * obj.at_pointer("/foo/a/1") == 20 + * obj.at_pointer("/foo")["a"].at(1) == 20 + * + * It is allowed for a key to be the empty string: + * + * dom::parser parser; + * object obj = parser.parse(R"({ "": { "a": [ 10, 20, 30 ] }})"_padded); + * obj.at_pointer("//a/1") == 20 + * obj.at_pointer("/")["a"].at(1) == 20 + * + * @return The value associated with the given JSON pointer, or: + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed + */ + inline simdjson_result at_pointer(std::string_view json_pointer) const noexcept; + + /** + * Recursive function which processes the JSON path of each child element + */ + inline void process_json_path_of_child_elements(std::vector::iterator& current, std::vector::iterator& end, const std::string_view& path_suffix, std::vector& accumulator) const noexcept; + + /** + * Adds support for JSONPath expression with wildcards '*' + */ + inline simdjson_result> at_path_with_wildcard(std::string_view json_path) const noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://www.rfc-editor.org/rfc/rfc9535 (RFC 9535) + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) const noexcept; + + /** + * Get the value associated with the given key. + * + * The key will be matched against **unescaped** JSON: + * + * dom::parser parser; + * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 + * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD + * + * This function has linear-time complexity: the keys are checked one by one. + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + */ + inline simdjson_result at_key(std::string_view key) const noexcept; + + /** + * Gets the values associated with keys of an object + * This function has linear-time complexity: the keys are checked one by one. + * + * @return the values associated with each key of an object + */ + inline std::vector& get_values(std::vector& out) const noexcept; + + /** + * Get the value associated with the given key in a case-insensitive manner. + * It is only guaranteed to work over ASCII inputs. + * + * Note: The key will be matched against **unescaped** JSON. + * + * This function has linear-time complexity: the keys are checked one by one. + * + * @return The value associated with this field, or: + * - NO_SUCH_FIELD if the field does not exist in the object + */ + inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; + + /** + * Implicitly convert object to element + */ + inline operator element() const noexcept; + +private: + simdjson_inline object(const internal::tape_ref &tape) noexcept; + + internal::tape_ref tape{}; + + friend class element; + friend struct simdjson_result; + template + friend class simdjson::internal::string_builder; +}; + +/** + * Key/value pair in an object. + */ +class key_value_pair { +public: + /** key in the key-value pair **/ + std::string_view key; + /** value in the key-value pair **/ + element value; + +private: + simdjson_inline key_value_pair(std::string_view _key, element _value) noexcept; + friend class object; +}; + +} // namespace dom + +/** The result of a JSON conversion that may fail. */ +template<> +struct simdjson_result : public internal::simdjson_result_base { +public: + simdjson_inline simdjson_result() noexcept; ///< @private + simdjson_inline simdjson_result(dom::object value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + + inline simdjson_result operator[](std::string_view key) const noexcept; + inline simdjson_result operator[](const char *key) const noexcept; + simdjson_result operator[](int) const noexcept = delete; + inline simdjson_result at_pointer(std::string_view json_pointer) const noexcept; + inline void process_json_path_of_child_elements(std::vector::iterator& current, std::vector::iterator& end, const std::string_view& path_suffix, std::vector& accumulator) const noexcept; + inline simdjson_result> at_path_with_wildcard(std::string_view json_path_new) const noexcept; + inline simdjson_result at_path(std::string_view json_path) const noexcept; + inline simdjson_result at_key(std::string_view key) const noexcept; + inline std::vector& get_values(std::vector& out) const noexcept; + inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; + +#if SIMDJSON_EXCEPTIONS + inline dom::object::iterator begin() const noexcept(false); + inline dom::object::iterator end() const noexcept(false); + inline size_t size() const noexcept(false); +#endif // SIMDJSON_EXCEPTIONS +}; + +} // namespace simdjson + +#if SIMDJSON_SUPPORTS_RANGES +namespace std { +namespace ranges { +template<> +inline constexpr bool enable_view = true; +#if SIMDJSON_EXCEPTIONS +template<> +inline constexpr bool enable_view> = true; +#endif // SIMDJSON_EXCEPTIONS +} // namespace ranges +} // namespace std +#endif // SIMDJSON_SUPPORTS_RANGES + +#endif // SIMDJSON_DOM_OBJECT_H +/* end file simdjson/dom/object.h */ +/* skipped duplicate #include "simdjson/dom/parser.h" */ +/* including simdjson/dom/serialization.h: #include "simdjson/dom/serialization.h" */ +/* begin file simdjson/dom/serialization.h */ +#ifndef SIMDJSON_SERIALIZATION_H +#define SIMDJSON_SERIALIZATION_H + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* skipped duplicate #include "simdjson/dom/element.h" */ +/* skipped duplicate #include "simdjson/dom/object.h" */ + +namespace simdjson { + +/** + * The string_builder template and mini_formatter class + * are not part of our public API and are subject to change + * at any time! + */ +namespace internal { + +template class base_formatter { +public: + /** Add a comma **/ + simdjson_inline void comma(); + /** Start an array, prints [ **/ + simdjson_inline void start_array(); + /** End an array, prints ] **/ + simdjson_inline void end_array(); + /** Start an array, prints { **/ + simdjson_inline void start_object(); + /** Start an array, prints } **/ + simdjson_inline void end_object(); + /** Prints a true **/ + simdjson_inline void true_atom(); + /** Prints a false **/ + simdjson_inline void false_atom(); + /** Prints a null **/ + simdjson_inline void null_atom(); + /** Prints a number **/ + simdjson_inline void number(int64_t x); + /** Prints a number **/ + simdjson_inline void number(uint64_t x); + /** Prints a number **/ + simdjson_inline void number(double x); + /** Prints a key (string + colon) **/ + simdjson_inline void key(std::string_view unescaped); + /** Prints a string. The string is escaped as needed. **/ + simdjson_inline void string(std::string_view unescaped); + /** Clears out the content. **/ + simdjson_inline void clear(); + /** + * Get access to the buffer, it is owned by the instance, but + * the user can make a copy. + **/ + simdjson_inline std::string_view str() const; + + /** Prints one character **/ + simdjson_inline void one_char(char c); + + /** Prints characters in [begin, end) verbatim. **/ + simdjson_inline void chars(const char *begin, const char *end); + + simdjson_inline void call_print_newline() { + static_cast(this)->print_newline(); + } + + simdjson_inline void call_print_indents(size_t depth) { + static_cast(this)->print_indents(depth); + } + + simdjson_inline void call_print_space() { + static_cast(this)->print_space(); + } + +protected: + // implementation details (subject to change) + /** Backing buffer **/ + struct vector_with_small_buffer { + vector_with_small_buffer() = default; + ~vector_with_small_buffer() { free_buffer(); } + + vector_with_small_buffer(const vector_with_small_buffer &) = delete; + vector_with_small_buffer & + operator=(const vector_with_small_buffer &) = delete; + + void clear() { + size = 0; + capacity = StaticCapacity; + free_buffer(); + buffer = array; + } + + simdjson_inline void push_back(char c) { + if (capacity < size + 1) + grow(capacity * 2); + buffer[size++] = c; + } + + simdjson_inline void append(const char *begin, const char *end) { + const size_t new_size = size + (end - begin); + if (capacity < new_size) + // std::max(new_size, capacity * 2); is broken in tests on Windows + grow(new_size < capacity * 2 ? capacity * 2 : new_size); + std::copy(begin, end, buffer + size); + size = new_size; + } + + std::string_view str() const { return std::string_view(buffer, size); } + + private: + void free_buffer() { + if (buffer != array) + delete[] buffer; + } + void grow(size_t new_capacity) { + auto new_buffer = new char[new_capacity]; + std::copy(buffer, buffer + size, new_buffer); + free_buffer(); + buffer = new_buffer; + capacity = new_capacity; + } + + static const size_t StaticCapacity = 64; + char array[StaticCapacity]; + char *buffer = array; + size_t size = 0; + size_t capacity = StaticCapacity; + } buffer{}; +}; + +/** + * @private This is the class that we expect to use with the string_builder + * template. It tries to produce a compact version of the JSON element + * as quickly as possible. + */ +class mini_formatter : public base_formatter { +public: + simdjson_inline void print_newline(); + + simdjson_inline void print_indents(size_t depth); + + simdjson_inline void print_space(); +}; + +class pretty_formatter : public base_formatter { +public: + simdjson_inline void print_newline(); + + simdjson_inline void print_indents(size_t depth); + + simdjson_inline void print_space(); + +protected: + int indent_step = 4; +}; + +/** + * @private The string_builder template allows us to construct + * a string from a document element. It is parametrized + * by a "formatter" which handles the details. Thus + * the string_builder template could support both minification + * and prettification, and various other tradeoffs. + * + * This is not to be confused with the simdjson::builder::string_builder + * which is a different class. + */ +template class string_builder { +public: + /** Construct an initially empty builder, would print the empty string **/ + string_builder() = default; + /** Append an element to the builder (to be printed) **/ + inline void append(simdjson::dom::element value); + /** Append an array to the builder (to be printed) **/ + inline void append(simdjson::dom::array value); + /** Append an object to the builder (to be printed) **/ + inline void append(simdjson::dom::object value); + /** Reset the builder (so that it would print the empty string) **/ + simdjson_inline void clear(); + /** + * Get access to the string. The string_view is owned by the builder + * and it is invalid to use it after the string_builder has been + * destroyed. + * However you can make a copy of the string_view on memory that you + * own. + */ + simdjson_inline std::string_view str() const; + /** Append a key_value_pair to the builder (to be printed) **/ + simdjson_inline void append(simdjson::dom::key_value_pair value); + +private: + formatter format{}; +}; + +} // namespace internal + +namespace dom { + +/** + * Print JSON to an output stream. + * + * @param out The output stream. + * @param value The element. + * @throw if there is an error with the underlying output stream. simdjson + * itself will not throw. + */ +inline std::ostream &operator<<(std::ostream &out, + simdjson::dom::element value); +#if SIMDJSON_EXCEPTIONS +inline std::ostream & +operator<<(std::ostream &out, + simdjson::simdjson_result x); +#endif +/** + * Print JSON to an output stream. + * + * @param out The output stream. + * @param value The array. + * @throw if there is an error with the underlying output stream. simdjson + * itself will not throw. + */ +inline std::ostream &operator<<(std::ostream &out, simdjson::dom::array value); +#if SIMDJSON_EXCEPTIONS +inline std::ostream & +operator<<(std::ostream &out, + simdjson::simdjson_result x); +#endif +/** + * Print JSON to an output stream. + * + * @param out The output stream. + * @param value The object. + * @throw if there is an error with the underlying output stream. simdjson + * itself will not throw. + */ +inline std::ostream &operator<<(std::ostream &out, simdjson::dom::object value); +#if SIMDJSON_EXCEPTIONS +inline std::ostream & +operator<<(std::ostream &out, + simdjson::simdjson_result x); +#endif +} // namespace dom + +/** + * Converts JSON to a string. + * + * dom::parser parser; + * element doc = parser.parse(" [ 1 , 2 , 3 ] "_padded); + * cout << to_string(doc) << endl; // prints [1,2,3] + * + */ +template std::string to_string(T x) { + // in C++, to_string is standard: + // http://www.cplusplus.com/reference/string/to_string/ Currently minify and + // to_string are identical but in the future, they may differ. + simdjson::internal::string_builder<> sb; + sb.append(x); + std::string_view answer = sb.str(); + return std::string(answer.data(), answer.size()); +} +#if SIMDJSON_EXCEPTIONS +template std::string to_string(simdjson_result x) { + if (x.error()) { + throw simdjson_error(x.error()); + } + return to_string(x.value()); +} +#endif + +/** + * Minifies a JSON element or document, printing the smallest possible valid + * JSON. + * + * dom::parser parser; + * element doc = parser.parse(" [ 1 , 2 , 3 ] "_padded); + * cout << minify(doc) << endl; // prints [1,2,3] + * + */ +template std::string minify(T x) { return to_string(x); } + +#if SIMDJSON_EXCEPTIONS +template std::string minify(simdjson_result x) { + if (x.error()) { + throw simdjson_error(x.error()); + } + return to_string(x.value()); +} +#endif + +/** + * Prettifies a JSON element or document, printing the valid JSON with + * indentation. + * + * dom::parser parser; + * element doc = parser.parse(" [ 1 , 2 , 3 ] "_padded); + * + * // Prints: + * // { + * // [ + * // 1, + * // 2, + * // 3 + * // ] + * // } + * cout << prettify(doc) << endl; + * + */ +template std::string prettify(T x) { + simdjson::internal::string_builder sb; + sb.append(x); + std::string_view answer = sb.str(); + return std::string(answer.data(), answer.size()); +} + +#if SIMDJSON_EXCEPTIONS +template std::string prettify(simdjson_result x) { + if (x.error()) { + throw simdjson_error(x.error()); + } + return to_string(x.value()); +} +#endif + +} // namespace simdjson + +#endif +/* end file simdjson/dom/serialization.h */ +/* including simdjson/dom/fractured_json.h: #include "simdjson/dom/fractured_json.h" */ +/* begin file simdjson/dom/fractured_json.h */ +#ifndef SIMDJSON_DOM_FRACTURED_JSON_H +#define SIMDJSON_DOM_FRACTURED_JSON_H + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* skipped duplicate #include "simdjson/dom/element.h" */ + +namespace simdjson { + +/** + * Configuration options for FracturedJson formatting. + * + * FracturedJson intelligently chooses between different layout strategies + * (inline, compact multiline, table, expanded) based on content complexity, + * length, and structure similarity. + */ +struct fractured_json_options { + /** + * Maximum total characters per line (default: 120). + * Content exceeding this will be expanded to multiple lines. + */ + size_t max_total_line_length = 120; + + /** + * Maximum length for inlined elements (default: 80). + * Simple arrays/objects shorter than this may be rendered inline. + */ + size_t max_inline_length = 80; + + /** + * Maximum nesting depth for inline rendering (default: 2). + * Elements with complexity exceeding this will be expanded. + * Complexity 0 = scalar, 1 = flat array/object, 2 = one level of nesting. + */ + size_t max_inline_complexity = 2; + + /** + * Maximum complexity for compact array formatting (default: 1). + * Arrays with elements of this complexity or less may have multiple + * items per line. + */ + size_t max_compact_array_complexity = 1; + + /** + * Number of spaces per indentation level (default: 4). + */ + size_t indent_spaces = 4; + + /** + * Enable tabular formatting for arrays of similar objects (default: true). + * When enabled, arrays of objects with identical keys are formatted + * as aligned tables. + */ + bool enable_table_format = true; + + /** + * Minimum number of rows to trigger table mode (default: 3). + */ + size_t min_table_rows = 3; + + /** + * Similarity threshold for table detection (default: 0.8). + * Objects must share at least this fraction of keys to be formatted + * as a table. + */ + double table_similarity_threshold = 0.8; + + /** + * Enable compact multiline arrays (default: true). + * When enabled, arrays of simple elements may have multiple items + * per line. + */ + bool enable_compact_multiline = true; + + /** + * Maximum array items per line in compact mode (default: 10). + */ + size_t max_items_per_line = 10; + + /** + * Add space inside brackets for simple containers (default: true). + * When true: { "key": "value" } + * When false: {"key": "value"} + */ + bool simple_bracket_padding = true; + + /** + * Add space after colons (default: true). + * When true: "key": "value" + * When false: "key":"value" + */ + bool colon_padding = true; + + /** + * Add space after commas in inline content (default: true). + * When true: [1, 2, 3] + * When false: [1,2,3] + */ + bool comma_padding = true; +}; + +/** + * Format JSON using FracturedJson formatting with default options. + * + * FracturedJson produces human-readable yet compact output by intelligently + * choosing between inline, compact multiline, table, and expanded layouts. + * + * dom::parser parser; + * element doc = parser.parse(json_string); + * cout << fractured_json(doc) << endl; + */ +template +std::string fractured_json(T x); + +/** + * Format JSON using FracturedJson formatting with custom options. + * + * dom::parser parser; + * element doc = parser.parse(json_string); + * fractured_json_options opts; + * opts.max_total_line_length = 80; + * cout << fractured_json(doc, opts) << endl; + */ +template +std::string fractured_json(T x, const fractured_json_options& options); + +#if SIMDJSON_EXCEPTIONS +template +std::string fractured_json(simdjson_result x); + +template +std::string fractured_json(simdjson_result x, const fractured_json_options& options); +#endif + +/** + * Format a JSON string using FracturedJson formatting. + * + * This is useful for formatting output from the builder/static reflection API + * or any valid JSON string. + * + * // With static reflection + * MyStruct data = {...}; + * auto minified = simdjson::to_json_string(data); + * auto formatted = simdjson::fractured_json_string(minified.value()); + * + * // Or with any JSON string + * std::string json = R"({"key":"value"})"; + * auto formatted = simdjson::fractured_json_string(json); + */ +inline std::string fractured_json_string(std::string_view json_str); + +/** + * Format a JSON string using FracturedJson formatting with custom options. + */ +inline std::string fractured_json_string(std::string_view json_str, + const fractured_json_options& options); + +} // namespace simdjson + +#endif // SIMDJSON_DOM_FRACTURED_JSON_H +/* end file simdjson/dom/fractured_json.h */ + +// Inline functions +/* including simdjson/dom/array-inl.h: #include "simdjson/dom/array-inl.h" */ +/* begin file simdjson/dom/array-inl.h */ +#ifndef SIMDJSON_ARRAY_INL_H +#define SIMDJSON_ARRAY_INL_H + +#include + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* skipped duplicate #include "simdjson/dom/array.h" */ +/* skipped duplicate #include "simdjson/dom/element.h" */ +/* skipped duplicate #include "simdjson/error-inl.h" */ +/* including simdjson/jsonpathutil.h: #include "simdjson/jsonpathutil.h" */ +/* begin file simdjson/jsonpathutil.h */ +#ifndef SIMDJSON_JSONPATHUTIL_H +#define SIMDJSON_JSONPATHUTIL_H + +#include +/* skipped duplicate #include "simdjson/common_defs.h" */ + +#include + +namespace simdjson { +/** + * Converts JSONPath to JSON Pointer. + * @param json_path The JSONPath string to be converted. + * @return A string containing the equivalent JSON Pointer. + */ +inline std::string json_path_to_pointer_conversion(std::string_view json_path) { + size_t i = 0; + // if JSONPath starts with $, skip it + // json_path.starts_with('$') requires C++20. + if (!json_path.empty() && json_path.front() == '$') { + i = 1; + } + if (i >= json_path.size() || (json_path[i] != '.' && + json_path[i] != '[')) { + return "-1"; // This is just a sentinel value, the caller should check for this and return an error. + } + + std::string result; + // Reserve space to reduce allocations, adjusting for potential increases due + // to escaping. + result.reserve(json_path.size() * 2); + + while (i < json_path.length()) { + if (json_path[i] == '.') { + result += '/'; + } else if (json_path[i] == '[') { + result += '/'; + ++i; // Move past the '[' + while (i < json_path.length() && json_path[i] != ']') { + if (json_path[i] == '~') { + result += "~0"; + } else if (json_path[i] == '/') { + result += "~1"; + } else { + result += json_path[i]; + } + ++i; + } + if (i == json_path.length() || json_path[i] != ']') { + return "-1"; // Using sentinel value that will be handled as an error by the caller. + } + } else { + if (json_path[i] == '~') { + result += "~0"; + } else if (json_path[i] == '/') { + result += "~1"; + } else { + result += json_path[i]; + } + } + ++i; + } + + return result; +} + +inline std::pair get_next_key_and_json_path(std::string_view& json_path) { + std::string_view key; + + if (json_path.empty()) { + return {key, json_path}; + } + size_t i = 0; + + // if JSONPath starts with $, skip it + if (json_path.front() == '$') { + i = 1; + } + + + if (i < json_path.length() && json_path[i] == '.') { + i += 1; + size_t key_start = i; + + while (i < json_path.length() && json_path[i] != '[' && json_path[i] != '.') { + ++i; + } + + key = json_path.substr(key_start, i - key_start); + } else if ((i+1 < json_path.size()) && json_path[i] == '[' && (json_path[i+1] == '\'' || json_path[i+1] == '"')) { + i += 2; + size_t key_start = i; + while (i < json_path.length() && json_path[i] != '\'' && json_path[i] != '"') { + ++i; + } + + key = json_path.substr(key_start, i - key_start); + + i += 2; + } else if ((i+2 < json_path.size()) && json_path[i] == '[' && json_path[i+1] == '*' && json_path[i+2] == ']') { // i.e [*].additional_keys or [*]["additional_keys"] + key = "*"; + i += 3; + } + + + return std::make_pair(key, json_path.substr(i)); +} + +} // namespace simdjson +#endif // SIMDJSON_JSONPATHUTIL_H +/* end file simdjson/jsonpathutil.h */ +/* including simdjson/internal/tape_ref-inl.h: #include "simdjson/internal/tape_ref-inl.h" */ +/* begin file simdjson/internal/tape_ref-inl.h */ +#ifndef SIMDJSON_TAPE_REF_INL_H +#define SIMDJSON_TAPE_REF_INL_H + +/* skipped duplicate #include "simdjson/dom/document.h" */ +/* skipped duplicate #include "simdjson/internal/tape_ref.h" */ +/* including simdjson/internal/tape_type.h: #include "simdjson/internal/tape_type.h" */ +/* begin file simdjson/internal/tape_type.h */ +#ifndef SIMDJSON_INTERNAL_TAPE_TYPE_H +#define SIMDJSON_INTERNAL_TAPE_TYPE_H + +namespace simdjson { +namespace internal { + +/** + * The possible types in the tape. + */ +enum class tape_type { + ROOT = 'r', + START_ARRAY = '[', + START_OBJECT = '{', + END_ARRAY = ']', + END_OBJECT = '}', + STRING = '"', + INT64 = 'l', + UINT64 = 'u', + DOUBLE = 'd', + TRUE_VALUE = 't', + FALSE_VALUE = 'f', + NULL_VALUE = 'n', + BIGINT = 'Z' // Big integer stored as string in string buffer +}; // enum class tape_type + +} // namespace internal +} // namespace simdjson + +#endif // SIMDJSON_INTERNAL_TAPE_TYPE_H +/* end file simdjson/internal/tape_type.h */ + +#include + +namespace simdjson { +namespace internal { + +constexpr const uint64_t JSON_VALUE_MASK = 0x00FFFFFFFFFFFFFF; +constexpr const uint32_t JSON_COUNT_MASK = 0xFFFFFF; + +// +// tape_ref inline implementation +// +simdjson_inline tape_ref::tape_ref() noexcept : doc{nullptr}, json_index{0} {} +simdjson_inline tape_ref::tape_ref(const dom::document *_doc, size_t _json_index) noexcept : doc{_doc}, json_index{_json_index} {} + + +simdjson_inline bool tape_ref::is_document_root() const noexcept { + return json_index == 1; // should we ever change the structure of the tape, this should get updated. +} +simdjson_inline bool tape_ref::usable() const noexcept { + return doc != nullptr; // when the document pointer is null, this tape_ref is uninitialized (should not be accessed). +} +// Some value types have a specific on-tape word value. It can be faster +// to check the type by doing a word-to-word comparison instead of extracting the +// most significant 8 bits. + +simdjson_inline bool tape_ref::is_double() const noexcept { + constexpr uint64_t tape_double = uint64_t(tape_type::DOUBLE)<<56; + return doc->tape[json_index] == tape_double; +} +simdjson_inline bool tape_ref::is_int64() const noexcept { + constexpr uint64_t tape_int64 = uint64_t(tape_type::INT64)<<56; + return doc->tape[json_index] == tape_int64; +} +simdjson_inline bool tape_ref::is_uint64() const noexcept { + constexpr uint64_t tape_uint64 = uint64_t(tape_type::UINT64)<<56; + return doc->tape[json_index] == tape_uint64; +} +simdjson_inline bool tape_ref::is_false() const noexcept { + constexpr uint64_t tape_false = uint64_t(tape_type::FALSE_VALUE)<<56; + return doc->tape[json_index] == tape_false; +} +simdjson_inline bool tape_ref::is_true() const noexcept { + constexpr uint64_t tape_true = uint64_t(tape_type::TRUE_VALUE)<<56; + return doc->tape[json_index] == tape_true; +} +simdjson_inline bool tape_ref::is_null_on_tape() const noexcept { + constexpr uint64_t tape_null = uint64_t(tape_type::NULL_VALUE)<<56; + return doc->tape[json_index] == tape_null; +} + +inline size_t tape_ref::after_element() const noexcept { + switch (tape_ref_type()) { + case tape_type::START_ARRAY: + case tape_type::START_OBJECT: + return matching_brace_index(); + case tape_type::UINT64: + case tape_type::INT64: + case tape_type::DOUBLE: + return json_index + 2; + default: + return json_index + 1; + } +} +simdjson_inline tape_type tape_ref::tape_ref_type() const noexcept { + return static_cast(doc->tape[json_index] >> 56); +} +simdjson_inline uint64_t internal::tape_ref::tape_value() const noexcept { + return doc->tape[json_index] & internal::JSON_VALUE_MASK; +} +simdjson_inline uint32_t internal::tape_ref::matching_brace_index() const noexcept { + return uint32_t(doc->tape[json_index]); +} +simdjson_inline uint32_t internal::tape_ref::scope_count() const noexcept { + return uint32_t((doc->tape[json_index] >> 32) & internal::JSON_COUNT_MASK); +} + +template +simdjson_inline T tape_ref::next_tape_value() const noexcept { + static_assert(sizeof(T) == sizeof(uint64_t), "next_tape_value() template parameter must be 64-bit"); + // Though the following is tempting... + // return *reinterpret_cast(&doc->tape[json_index + 1]); + // It is not generally safe. It is safer, and often faster to rely + // on memcpy. Yes, it is uglier, but it is also encapsulated. + T x; + std::memcpy(&x,&doc->tape[json_index + 1],sizeof(uint64_t)); + return x; +} + +simdjson_inline uint32_t internal::tape_ref::get_string_length() const noexcept { + size_t string_buf_index = size_t(tape_value()); + uint32_t len; + std::memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len)); + return len; +} + +simdjson_inline const char * internal::tape_ref::get_c_str() const noexcept { + size_t string_buf_index = size_t(tape_value()); + return reinterpret_cast(&doc->string_buf[string_buf_index + sizeof(uint32_t)]); +} + +inline std::string_view internal::tape_ref::get_string_view() const noexcept { + return std::string_view( + get_c_str(), + get_string_length() + ); +} + +} // namespace internal +} // namespace simdjson + +#endif // SIMDJSON_TAPE_REF_INL_H +/* end file simdjson/internal/tape_ref-inl.h */ + +#include + +namespace simdjson { + +// +// simdjson_result inline implementation +// +simdjson_inline simdjson_result::simdjson_result() noexcept + : internal::simdjson_result_base() {} +simdjson_inline simdjson_result::simdjson_result(dom::array value) noexcept + : internal::simdjson_result_base(std::forward(value)) {} +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : internal::simdjson_result_base(error) {} + +#if SIMDJSON_EXCEPTIONS + +inline dom::array::iterator simdjson_result::begin() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.begin(); +} +inline dom::array::iterator simdjson_result::end() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.end(); +} +inline size_t simdjson_result::size() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.size(); +} + +#endif // SIMDJSON_EXCEPTIONS + +inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) const noexcept { + if (error()) { return error(); } + return first.at_pointer(json_pointer); +} + + inline simdjson_result simdjson_result::at_path(std::string_view json_path) const noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { return INVALID_JSON_POINTER; } + return at_pointer(json_pointer); + } + +inline simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) const noexcept { + if (error()) { + return error(); + } + return first.at_path_with_wildcard(json_path); +} + +inline simdjson_result simdjson_result::at(size_t index) const noexcept { + if (error()) { return error(); } + return first.at(index); +} + +inline std::vector& simdjson_result::get_values(std::vector& out) const noexcept { + return first.get_values(out); +} + +namespace dom { + +// +// array inline implementation +// +simdjson_inline array::array() noexcept : tape{} {} +simdjson_inline array::array(const internal::tape_ref &_tape) noexcept : tape{_tape} {} +inline array::iterator array::begin() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + return internal::tape_ref(tape.doc, tape.json_index + 1); +} +inline array::iterator array::end() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + return internal::tape_ref(tape.doc, tape.after_element() - 1); +} +inline size_t array::size() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + return tape.scope_count(); +} +inline size_t array::number_of_slots() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + return tape.matching_brace_index() - tape.json_index; +} +inline simdjson_result array::at_pointer(std::string_view json_pointer) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + if(json_pointer.empty()) { // an empty string means that we return the current node + return element(this->tape); // copy the current node + } else if(json_pointer[0] != '/') { // otherwise there is an error + return INVALID_JSON_POINTER; + } + json_pointer = json_pointer.substr(1); + // - means "the append position" or "the element after the end of the array" + // We don't support this, because we're returning a real element, not a position. + if (json_pointer == "-") { return INDEX_OUT_OF_BOUNDS; } + + // Read the array index + size_t array_index = 0; + size_t i; + for (i = 0; i < json_pointer.length() && json_pointer[i] != '/'; i++) { + uint8_t digit = uint8_t(json_pointer[i] - '0'); + // Check for non-digit in array index. If it's there, we're trying to get a field in an object + if (digit > 9) { return INCORRECT_TYPE; } + array_index = array_index*10 + digit; + } + + // 0 followed by other digits is invalid + if (i > 1 && json_pointer[0] == '0') { return INVALID_JSON_POINTER; } // "JSON pointer array index has other characters after 0" + + // Empty string is invalid; so is a "/" with no digits before it + if (i == 0) { return INVALID_JSON_POINTER; } // "Empty string in JSON pointer array index" + + // Get the child + auto child = array(tape).at(array_index); + // If there is an error, it ends here + if(child.error()) { + return child; + } + // If there is a /, we're not done yet, call recursively. + if (i < json_pointer.length()) { + child = child.at_pointer(json_pointer.substr(i)); + } + return child; +} + +inline simdjson_result array::at_path(std::string_view json_path) const noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { return INVALID_JSON_POINTER; } + return at_pointer(json_pointer); +} + +inline void array::process_json_path_of_child_elements(std::vector::iterator& current, std::vector::iterator& end, const std::string_view& path_suffix, std::vector& accumulator) const noexcept { + if (current == end) { + return; + } + + simdjson_result> result; + + + for (auto it = current; it != end; ++it) { + std::vector child_result; + auto error = it->at_path_with_wildcard(path_suffix).get(child_result); + if(error) { + continue; + } + accumulator.reserve(accumulator.size() + child_result.size()); + accumulator.insert(accumulator.end(), + std::make_move_iterator(child_result.begin()), + std::make_move_iterator(child_result.end())); + } +} + +inline simdjson_result> array::at_path_with_wildcard(std::string_view json_path) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + + size_t i = 0; + // json_path.starts_with('$') requires C++20. + if (!json_path.empty() && json_path.front() == '$') { + i = 1; + } + + if (i >= json_path.size() || (json_path[i] != '.' && json_path[i] != '[')) { + return INVALID_JSON_POINTER; + } + + if (json_path.find("*") != std::string::npos) { + std::vector child_values; + + if ( + (json_path.compare(i, 3, "[*]") == 0 && json_path.size() == i + 3) || + (json_path.compare(i, 2,".*") == 0 && json_path.size() == i + 2) + ) { + get_values(child_values); + return child_values; + } + + std::pair key_and_json_path = get_next_key_and_json_path(json_path); + + std::string_view key = key_and_json_path.first; + json_path = key_and_json_path.second; + + if (key.size() > 0) { + if (key == "*") { + get_values(child_values); + } else { + element pointer_result; + std::string json_pointer = std::string("/") + std::string(key); + auto error = at_pointer(json_pointer).get(pointer_result); + + if (!error) { + child_values.emplace_back(pointer_result); + } + } + + std::vector result = {}; + + if (child_values.size() > 0) { + std::vector::iterator child_values_begin = child_values.begin(); + std::vector::iterator child_values_end = child_values.end(); + + process_json_path_of_child_elements(child_values_begin, child_values_end, json_path, result); + } + + return result; + } else { + return INVALID_JSON_POINTER; + } + } else { + element result; + auto error = at_path(json_path).get(result); + if (error) { + return error; + } + + return std::vector{std::move(result)}; + } +} + +inline simdjson_result array::at(size_t index) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + size_t i=0; + for (auto element : *this) { + if (i == index) { return element; } + i++; + } + return INDEX_OUT_OF_BOUNDS; +} + +inline std::vector& array::get_values(std::vector& out) const noexcept { + out.reserve(this->size()); + for (auto element : *this) { + out.emplace_back(element); + } + + return out; +} + +inline array::operator element() const noexcept { + return element(tape); +} + +// +// array::iterator inline implementation +// +simdjson_inline array::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } +inline element array::iterator::operator*() const noexcept { + return element(tape); +} +inline array::iterator& array::iterator::operator++() noexcept { + tape.json_index = tape.after_element(); + return *this; +} +inline array::iterator array::iterator::operator++(int) noexcept { + array::iterator out = *this; + ++*this; + return out; +} +inline bool array::iterator::operator!=(const array::iterator& other) const noexcept { + return tape.json_index != other.tape.json_index; +} +inline bool array::iterator::operator==(const array::iterator& other) const noexcept { + return tape.json_index == other.tape.json_index; +} +inline bool array::iterator::operator<(const array::iterator& other) const noexcept { + return tape.json_index < other.tape.json_index; +} +inline bool array::iterator::operator<=(const array::iterator& other) const noexcept { + return tape.json_index <= other.tape.json_index; +} +inline bool array::iterator::operator>=(const array::iterator& other) const noexcept { + return tape.json_index >= other.tape.json_index; +} +inline bool array::iterator::operator>(const array::iterator& other) const noexcept { + return tape.json_index > other.tape.json_index; +} + +} // namespace dom + + +} // namespace simdjson + +/* including simdjson/dom/element-inl.h: #include "simdjson/dom/element-inl.h" */ +/* begin file simdjson/dom/element-inl.h */ +#ifndef SIMDJSON_ELEMENT_INL_H +#define SIMDJSON_ELEMENT_INL_H + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* skipped duplicate #include "simdjson/dom/element.h" */ +/* skipped duplicate #include "simdjson/dom/document.h" */ +/* skipped duplicate #include "simdjson/dom/object.h" */ +/* skipped duplicate #include "simdjson/internal/tape_type.h" */ + +/* including simdjson/dom/object-inl.h: #include "simdjson/dom/object-inl.h" */ +/* begin file simdjson/dom/object-inl.h */ +#ifndef SIMDJSON_OBJECT_INL_H +#define SIMDJSON_OBJECT_INL_H + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* skipped duplicate #include "simdjson/dom/object.h" */ +/* skipped duplicate #include "simdjson/dom/document.h" */ + +/* skipped duplicate #include "simdjson/dom/element-inl.h" */ +/* skipped duplicate #include "simdjson/error-inl.h" */ +/* skipped duplicate #include "simdjson/jsonpathutil.h" */ + +#include + +namespace simdjson { + +// +// simdjson_result inline implementation +// +simdjson_inline simdjson_result::simdjson_result() noexcept + : internal::simdjson_result_base() {} +simdjson_inline simdjson_result::simdjson_result(dom::object value) noexcept + : internal::simdjson_result_base(std::forward(value)) {} +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : internal::simdjson_result_base(error) {} + +inline simdjson_result simdjson_result::operator[](std::string_view key) const noexcept { + if (error()) { return error(); } + return first[key]; +} +inline simdjson_result simdjson_result::operator[](const char *key) const noexcept { + if (error()) { return error(); } + return first[key]; +} +inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) const noexcept { + if (error()) { return error(); } + return first.at_pointer(json_pointer); +} +inline simdjson_result simdjson_result::at_path(std::string_view json_path) const noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { return INVALID_JSON_POINTER; } + return at_pointer(json_pointer); +} +inline simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) const noexcept { + if (error()) { + return error(); + } + return first.at_path_with_wildcard(json_path); +} +inline simdjson_result simdjson_result::at_key(std::string_view key) const noexcept { + if (error()) { return error(); } + return first.at_key(key); +} +inline std::vector& simdjson_result::get_values(std::vector& out) const noexcept { + return first.get_values(out); +} +inline simdjson_result simdjson_result::at_key_case_insensitive(std::string_view key) const noexcept { + if (error()) { return error(); } + return first.at_key_case_insensitive(key); +} + +#if SIMDJSON_EXCEPTIONS + +inline dom::object::iterator simdjson_result::begin() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.begin(); +} +inline dom::object::iterator simdjson_result::end() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.end(); +} +inline size_t simdjson_result::size() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.size(); +} + +#endif // SIMDJSON_EXCEPTIONS + +namespace dom { + +// +// object inline implementation +// +simdjson_inline object::object() noexcept : tape{} {} +simdjson_inline object::object(const internal::tape_ref &_tape) noexcept : tape{_tape} { } +inline object::iterator object::begin() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + return internal::tape_ref(tape.doc, tape.json_index + 1); +} +inline object::iterator object::end() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + return internal::tape_ref(tape.doc, tape.after_element() - 1); +} +inline size_t object::size() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + return tape.scope_count(); +} + +inline simdjson_result object::operator[](std::string_view key) const noexcept { + return at_key(key); +} +inline simdjson_result object::operator[](const char *key) const noexcept { + return at_key(key); +} +inline simdjson_result object::at_pointer(std::string_view json_pointer) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + if(json_pointer.empty()) { // an empty string means that we return the current node + return element(this->tape); // copy the current node + } else if(json_pointer[0] != '/') { // otherwise there is an error + return INVALID_JSON_POINTER; + } + json_pointer = json_pointer.substr(1); + size_t slash = json_pointer.find('/'); + std::string_view key = json_pointer.substr(0, slash); + // Grab the child with the given key + simdjson_result child; + + // If there is an escape character in the key, unescape it and then get the child. + size_t escape = key.find('~'); + if (escape != std::string_view::npos) { + // Unescape the key + std::string unescaped(key); + do { + switch (unescaped[escape+1]) { + case '0': + unescaped.replace(escape, 2, "~"); + break; + case '1': + unescaped.replace(escape, 2, "/"); + break; + default: + return INVALID_JSON_POINTER; // "Unexpected ~ escape character in JSON pointer"); + } + escape = unescaped.find('~', escape+1); + } while (escape != std::string::npos); + child = at_key(unescaped); + } else { + child = at_key(key); + } + if(child.error()) { + return child; // we do not continue if there was an error + } + // If there is a /, we have to recurse and look up more of the path + if (slash != std::string_view::npos) { + child = child.at_pointer(json_pointer.substr(slash)); + } + return child; +} + +inline simdjson_result object::at_path(std::string_view json_path) const noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { return INVALID_JSON_POINTER; } + return at_pointer(json_pointer); +} + +inline void object::process_json_path_of_child_elements(std::vector::iterator& current, std::vector::iterator& end, const std::string_view& path_suffix, std::vector& accumulator) const noexcept { + if (current == end) { + return; + } + + simdjson_result> result; + + for (auto it = current; it != end; ++it) { + std::vector child_result; + auto error = it->at_path_with_wildcard(path_suffix).get(child_result); + if(error) { + continue; + } + accumulator.reserve(accumulator.size() + child_result.size()); + accumulator.insert(accumulator.end(), + std::make_move_iterator(child_result.begin()), + std::make_move_iterator(child_result.end())); + } +} + +inline simdjson_result> object::at_path_with_wildcard(std::string_view json_path) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + + size_t i = 0; + if (json_path.empty()) { + return INVALID_JSON_POINTER; + } + // if JSONPath starts with $, skip it + // json_path.starts_with('$') requires C++20. + if (json_path.front() == '$') { + i = 1; + } + + if (i >= json_path.size() || (json_path[i] != '.' && json_path[i] != '[')) { + // expect JSONPath expressions to always start with $ but this isn't currently + // expected in jsonpathutil.h. + return INVALID_JSON_POINTER; + } + + if (json_path.find("*") != std::string::npos) { + + std::vector child_values; + + if ( + (json_path.compare(i, 3, "[*]") == 0 && json_path.size() == i + 3) || + (json_path.compare(i, 2,".*") == 0 && json_path.size() == i + 2) + ) { + get_values(child_values); + return child_values; + } + + std::pair key_and_json_path = get_next_key_and_json_path(json_path); + + std::string_view key = key_and_json_path.first; + json_path = key_and_json_path.second; + + if (key.size() > 0) { + if (key == "*") { + get_values(child_values); + } else { + element pointer_result; + auto error = at_pointer(std::string("/") + std::string(key)).get(pointer_result); + + if (!error) { + child_values.emplace_back(pointer_result); + } + } + + std::vector result = {}; + if (child_values.size() > 0) { + + std::vector::iterator child_values_begin = child_values.begin(); + std::vector::iterator child_values_end = child_values.end(); + + process_json_path_of_child_elements(child_values_begin, child_values_end, json_path, result); + } + + return result; + } else { + return INVALID_JSON_POINTER; + } + } else { + element result; + auto error = this->at_path(json_path).get(result); + if (error) { + return error; + } + return std::vector{std::move(result)}; + } +} + +inline simdjson_result object::at_key(std::string_view key) const noexcept { + iterator end_field = end(); + for (iterator field = begin(); field != end_field; ++field) { + if (field.key_equals(key)) { + return field.value(); + } + } + return NO_SUCH_FIELD; +} + +inline std::vector& object::get_values(std::vector& out) const noexcept { + iterator end_field = end(); + iterator begin_field = begin(); + + out.reserve(std::distance(begin_field, end_field)); + for (iterator field = begin_field; field != end_field; ++field) { + out.emplace_back(field.value()); + } + + return out; +} +// In case you wonder why we need this, please see +// https://github.com/simdjson/simdjson/issues/323 +// People do seek keys in a case-insensitive manner. +inline simdjson_result object::at_key_case_insensitive(std::string_view key) const noexcept { + iterator end_field = end(); + for (iterator field = begin(); field != end_field; ++field) { + if (field.key_equals_case_insensitive(key)) { + return field.value(); + } + } + return NO_SUCH_FIELD; +} + +inline object::operator element() const noexcept { + return element(tape); +} + +// +// object::iterator inline implementation +// +simdjson_inline object::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } +inline const key_value_pair object::iterator::operator*() const noexcept { + return key_value_pair(key(), value()); +} +inline bool object::iterator::operator!=(const object::iterator& other) const noexcept { + return tape.json_index != other.tape.json_index; +} +inline bool object::iterator::operator==(const object::iterator& other) const noexcept { + return tape.json_index == other.tape.json_index; +} +inline bool object::iterator::operator<(const object::iterator& other) const noexcept { + return tape.json_index < other.tape.json_index; +} +inline bool object::iterator::operator<=(const object::iterator& other) const noexcept { + return tape.json_index <= other.tape.json_index; +} +inline bool object::iterator::operator>=(const object::iterator& other) const noexcept { + return tape.json_index >= other.tape.json_index; +} +inline bool object::iterator::operator>(const object::iterator& other) const noexcept { + return tape.json_index > other.tape.json_index; +} +inline object::iterator& object::iterator::operator++() noexcept { + tape.json_index++; + tape.json_index = tape.after_element(); + return *this; +} +inline object::iterator object::iterator::operator++(int) noexcept { + object::iterator out = *this; + ++*this; + return out; +} +inline std::string_view object::iterator::key() const noexcept { + return tape.get_string_view(); +} +inline uint32_t object::iterator::key_length() const noexcept { + return tape.get_string_length(); +} +inline const char* object::iterator::key_c_str() const noexcept { + return reinterpret_cast(&tape.doc->string_buf[size_t(tape.tape_value()) + sizeof(uint32_t)]); +} +inline element object::iterator::value() const noexcept { + return element(internal::tape_ref(tape.doc, tape.json_index + 1)); +} + +/** + * Design notes: + * Instead of constructing a string_view and then comparing it with a + * user-provided strings, it is probably more performant to have dedicated + * functions taking as a parameter the string we want to compare against + * and return true when they are equal. That avoids the creation of a temporary + * std::string_view. Though it is possible for the compiler to avoid entirely + * any overhead due to string_view, relying too much on compiler magic is + * problematic: compiler magic sometimes fail, and then what do you do? + * Also, enticing users to rely on high-performance function is probably better + * on the long run. + */ + +inline bool object::iterator::key_equals(std::string_view o) const noexcept { + // We use the fact that the key length can be computed quickly + // without access to the string buffer. + const uint32_t len = key_length(); + if(o.size() == len) { + // We avoid construction of a temporary string_view instance. + return (memcmp(o.data(), key_c_str(), len) == 0); + } + return false; +} + +inline bool object::iterator::key_equals_case_insensitive(std::string_view o) const noexcept { + // We use the fact that the key length can be computed quickly + // without access to the string buffer. + const uint32_t len = key_length(); + if(o.size() == len) { + // See For case-insensitive string comparisons, avoid char-by-char functions + // https://lemire.me/blog/2020/04/30/for-case-insensitive-string-comparisons-avoid-char-by-char-functions/ + // Note that it might be worth rolling our own strncasecmp function, with vectorization. + return (simdjson_strncasecmp(o.data(), key_c_str(), len) == 0); + } + return false; +} +// +// key_value_pair inline implementation +// +inline key_value_pair::key_value_pair(std::string_view _key, element _value) noexcept : + key(_key), value(_value) {} + +} // namespace dom + +} // namespace simdjson + +#if SIMDJSON_SUPPORTS_RANGES +static_assert(std::ranges::view); +static_assert(std::ranges::sized_range); +#if SIMDJSON_EXCEPTIONS +static_assert(std::ranges::view>); +static_assert(std::ranges::sized_range>); +#endif // SIMDJSON_EXCEPTIONS +#endif // SIMDJSON_SUPPORTS_RANGES + +#endif // SIMDJSON_OBJECT_INL_H +/* end file simdjson/dom/object-inl.h */ +/* skipped duplicate #include "simdjson/error-inl.h" */ +/* skipped duplicate #include "simdjson/jsonpathutil.h" */ + +#include +#include + +namespace simdjson { + +// +// simdjson_result inline implementation +// +simdjson_inline simdjson_result::simdjson_result() noexcept + : internal::simdjson_result_base() {} +simdjson_inline simdjson_result::simdjson_result(dom::element &&value) noexcept + : internal::simdjson_result_base(std::forward(value)) {} +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : internal::simdjson_result_base(error) {} +inline simdjson_result simdjson_result::type() const noexcept { + if (error()) { return error(); } + return first.type(); +} + +template +simdjson_inline bool simdjson_result::is() const noexcept { + return !error() && first.is(); +} +template +simdjson_inline simdjson_result simdjson_result::get() const noexcept { + if (error()) { return error(); } + return first.get(); +} +template +simdjson_warn_unused simdjson_inline error_code simdjson_result::get(T &value) const noexcept { + if (error()) { return error(); } + return first.get(value); +} + +simdjson_inline simdjson_result simdjson_result::get_array() const noexcept { + if (error()) { return error(); } + return first.get_array(); +} +simdjson_inline simdjson_result simdjson_result::get_object() const noexcept { + if (error()) { return error(); } + return first.get_object(); +} +simdjson_inline simdjson_result simdjson_result::get_c_str() const noexcept { + if (error()) { return error(); } + return first.get_c_str(); +} +simdjson_inline simdjson_result simdjson_result::get_string_length() const noexcept { + if (error()) { return error(); } + return first.get_string_length(); +} +simdjson_inline simdjson_result simdjson_result::get_string() const noexcept { + if (error()) { return error(); } + return first.get_string(); +} +simdjson_inline simdjson_result simdjson_result::get_int64() const noexcept { + if (error()) { return error(); } + return first.get_int64(); +} +simdjson_inline simdjson_result simdjson_result::get_uint64() const noexcept { + if (error()) { return error(); } + return first.get_uint64(); +} +simdjson_inline simdjson_result simdjson_result::get_double() const noexcept { + if (error()) { return error(); } + return first.get_double(); +} +simdjson_inline simdjson_result simdjson_result::get_bool() const noexcept { + if (error()) { return error(); } + return first.get_bool(); +} +simdjson_inline simdjson_result simdjson_result::get_bigint() const noexcept { + if (error()) { return error(); } + return first.get_bigint(); +} + +simdjson_inline bool simdjson_result::is_array() const noexcept { + return !error() && first.is_array(); +} +simdjson_inline bool simdjson_result::is_object() const noexcept { + return !error() && first.is_object(); +} +simdjson_inline bool simdjson_result::is_string() const noexcept { + return !error() && first.is_string(); +} +simdjson_inline bool simdjson_result::is_int64() const noexcept { + return !error() && first.is_int64(); +} +simdjson_inline bool simdjson_result::is_uint64() const noexcept { + return !error() && first.is_uint64(); +} +simdjson_inline bool simdjson_result::is_double() const noexcept { + return !error() && first.is_double(); +} +simdjson_inline bool simdjson_result::is_number() const noexcept { + return !error() && first.is_number(); +} +simdjson_inline bool simdjson_result::is_bool() const noexcept { + return !error() && first.is_bool(); +} + +simdjson_inline bool simdjson_result::is_null() const noexcept { + return !error() && first.is_null(); +} +simdjson_inline bool simdjson_result::is_bigint() const noexcept { + return !error() && first.is_bigint(); +} + +simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) const noexcept { + if (error()) { return error(); } + return first[key]; +} +simdjson_inline simdjson_result simdjson_result::operator[](const char *key) const noexcept { + if (error()) { return error(); } + return first[key]; +} +simdjson_inline simdjson_result simdjson_result::at_pointer(const std::string_view json_pointer) const noexcept { + if (error()) { return error(); } + return first.at_pointer(json_pointer); +} +simdjson_inline simdjson_result simdjson_result::at_path(const std::string_view json_path) const noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { return INVALID_JSON_POINTER; } + return at_pointer(json_pointer); +} + +simdjson_inline simdjson_result> simdjson_result::at_path_with_wildcard(const std::string_view json_path) const noexcept { + if (error()) { return error(); } + return first.at_path_with_wildcard(json_path); +} + +#ifndef SIMDJSON_DISABLE_DEPRECATED_API +[[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] +simdjson_inline simdjson_result simdjson_result::at(const std::string_view json_pointer) const noexcept { +SIMDJSON_PUSH_DISABLE_WARNINGS +SIMDJSON_DISABLE_DEPRECATED_WARNING + if (error()) { return error(); } + return first.at(json_pointer); +SIMDJSON_POP_DISABLE_WARNINGS +} +#endif // SIMDJSON_DISABLE_DEPRECATED_API +simdjson_inline simdjson_result simdjson_result::at(size_t index) const noexcept { + if (error()) { return error(); } + return first.at(index); +} +simdjson_inline simdjson_result simdjson_result::at_key(std::string_view key) const noexcept { + if (error()) { return error(); } + return first.at_key(key); +} +simdjson_inline simdjson_result simdjson_result::at_key_case_insensitive(std::string_view key) const noexcept { + if (error()) { return error(); } + return first.at_key_case_insensitive(key); +} + +#if SIMDJSON_EXCEPTIONS + +simdjson_inline simdjson_result::operator bool() const noexcept(false) { + return get(); +} +simdjson_inline simdjson_result::operator const char *() const noexcept(false) { + return get(); +} +simdjson_inline simdjson_result::operator std::string_view() const noexcept(false) { + return get(); +} +simdjson_inline simdjson_result::operator uint64_t() const noexcept(false) { + return get(); +} +simdjson_inline simdjson_result::operator int64_t() const noexcept(false) { + return get(); +} +simdjson_inline simdjson_result::operator double() const noexcept(false) { + return get(); +} +simdjson_inline simdjson_result::operator dom::array() const noexcept(false) { + return get(); +} +simdjson_inline simdjson_result::operator dom::object() const noexcept(false) { + return get(); +} + +simdjson_inline dom::array::iterator simdjson_result::begin() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.begin(); +} +simdjson_inline dom::array::iterator simdjson_result::end() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.end(); +} + +#endif // SIMDJSON_EXCEPTIONS + +namespace dom { + +// +// element inline implementation +// +simdjson_inline element::element() noexcept : tape{} {} +simdjson_inline element::element(const internal::tape_ref &_tape) noexcept : tape{_tape} { } + +inline element_type element::type() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + auto tape_type = tape.tape_ref_type(); + return tape_type == internal::tape_type::FALSE_VALUE ? element_type::BOOL : static_cast(tape_type); +} + +inline simdjson_result element::get_bool() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + if(tape.is_true()) { + return true; + } else if(tape.is_false()) { + return false; + } + return INCORRECT_TYPE; +} +inline simdjson_result element::get_bigint() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); + switch (tape.tape_ref_type()) { + case internal::tape_type::BIGINT: + return tape.get_string_view(); + default: + return INCORRECT_TYPE; + } +} +inline simdjson_result element::get_c_str() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + switch (tape.tape_ref_type()) { + case internal::tape_type::STRING: { + return tape.get_c_str(); + } + default: + return INCORRECT_TYPE; + } +} +inline simdjson_result element::get_string_length() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + switch (tape.tape_ref_type()) { + case internal::tape_type::STRING: { + return tape.get_string_length(); + } + default: + return INCORRECT_TYPE; + } +} +inline simdjson_result element::get_string() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + switch (tape.tape_ref_type()) { + case internal::tape_type::STRING: + return tape.get_string_view(); + default: + return INCORRECT_TYPE; + } +} +inline simdjson_result element::get_uint64() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + if(simdjson_unlikely(!tape.is_uint64())) { // branch rarely taken + if(tape.is_int64()) { + int64_t result = tape.next_tape_value(); + if (result < 0) { + return NUMBER_OUT_OF_RANGE; + } + return uint64_t(result); + } + return INCORRECT_TYPE; + } + return tape.next_tape_value(); +} +inline simdjson_result element::get_int64() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + if(simdjson_unlikely(!tape.is_int64())) { // branch rarely taken + if(tape.is_uint64()) { + uint64_t result = tape.next_tape_value(); + // Wrapping max in parens to handle Windows issue: https://stackoverflow.com/questions/11544073/how-do-i-deal-with-the-max-macro-in-windows-h-colliding-with-max-in-std + if (result > uint64_t((std::numeric_limits::max)())) { + return NUMBER_OUT_OF_RANGE; + } + return static_cast(result); + } + return INCORRECT_TYPE; + } + return tape.next_tape_value(); +} +inline simdjson_result element::get_double() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + // Performance considerations: + // 1. Querying tape_ref_type() implies doing a shift, it is fast to just do a straight + // comparison. + // 2. Using a switch-case relies on the compiler guessing what kind of code generation + // we want... But the compiler cannot know that we expect the type to be "double" + // most of the time. + // We can expect get to refer to a double type almost all the time. + // It is important to craft the code accordingly so that the compiler can use this + // information. (This could also be solved with profile-guided optimization.) + if(simdjson_unlikely(!tape.is_double())) { // branch rarely taken + if(tape.is_uint64()) { + return double(tape.next_tape_value()); + } else if(tape.is_int64()) { + return double(tape.next_tape_value()); + } + return INCORRECT_TYPE; + } + // this is common: + return tape.next_tape_value(); +} +inline simdjson_result element::get_array() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + switch (tape.tape_ref_type()) { + case internal::tape_type::START_ARRAY: + return array(tape); + default: + return INCORRECT_TYPE; + } +} +inline simdjson_result element::get_object() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + switch (tape.tape_ref_type()) { + case internal::tape_type::START_OBJECT: + return object(tape); + default: + return INCORRECT_TYPE; + } +} + +template +simdjson_warn_unused simdjson_inline error_code element::get(T &value) const noexcept { + return get().get(value); +} +// An element-specific version prevents recursion with simdjson_result::get(value) +template<> +simdjson_warn_unused simdjson_inline error_code element::get(element &value) const noexcept { + value = element(tape); + return SUCCESS; +} +template +inline void element::tie(T &value, error_code &error) && noexcept { + error = get(value); +} + +template +simdjson_inline bool element::is() const noexcept { + auto result = get(); + return !result.error(); +} + +template<> inline simdjson_result element::get() const noexcept { return get_array(); } +template<> inline simdjson_result element::get() const noexcept { return get_object(); } +template<> inline simdjson_result element::get() const noexcept { return get_c_str(); } +template<> inline simdjson_result element::get() const noexcept { return get_string(); } +template<> inline simdjson_result element::get() const noexcept { return get_int64(); } +template<> inline simdjson_result element::get() const noexcept { return get_uint64(); } +template<> inline simdjson_result element::get() const noexcept { return get_double(); } +template<> inline simdjson_result element::get() const noexcept { return get_bool(); } + +inline bool element::is_array() const noexcept { return is(); } +inline bool element::is_object() const noexcept { return is(); } +inline bool element::is_string() const noexcept { return is(); } +inline bool element::is_int64() const noexcept { return is(); } +inline bool element::is_uint64() const noexcept { return is(); } +inline bool element::is_double() const noexcept { return is(); } +inline bool element::is_bool() const noexcept { return is(); } +inline bool element::is_number() const noexcept { return is_int64() || is_uint64() || is_double(); } + +inline bool element::is_null() const noexcept { + return tape.is_null_on_tape(); +} + +inline bool element::is_bigint() const noexcept { + return tape.tape_ref_type() == internal::tape_type::BIGINT; +} + +#if SIMDJSON_EXCEPTIONS + +inline element::operator bool() const noexcept(false) { return get(); } +inline element::operator const char*() const noexcept(false) { return get(); } +inline element::operator std::string_view() const noexcept(false) { return get(); } +inline element::operator uint64_t() const noexcept(false) { return get(); } +inline element::operator int64_t() const noexcept(false) { return get(); } +inline element::operator double() const noexcept(false) { return get(); } +inline element::operator array() const noexcept(false) { return get(); } +inline element::operator object() const noexcept(false) { return get(); } + +inline array::iterator element::begin() const noexcept(false) { + return get().begin(); +} +inline array::iterator element::end() const noexcept(false) { + return get().end(); +} + +#endif // SIMDJSON_EXCEPTIONS + +inline simdjson_result element::operator[](std::string_view key) const noexcept { + return at_key(key); +} +inline simdjson_result element::operator[](const char *key) const noexcept { + return at_key(key); +} + +inline bool is_pointer_well_formed(std::string_view json_pointer) noexcept { + if (simdjson_unlikely(json_pointer[0] != '/')) { + return false; + } + size_t escape = json_pointer.find('~'); + if (escape == std::string_view::npos) { + return true; + } + if (escape == json_pointer.size() - 1) { + return false; + } + if (json_pointer[escape + 1] != '0' && json_pointer[escape + 1] != '1') { + return false; + } + return true; +} + +inline simdjson_result element::at_pointer(std::string_view json_pointer) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + switch (tape.tape_ref_type()) { + case internal::tape_type::START_OBJECT: + return object(tape).at_pointer(json_pointer); + case internal::tape_type::START_ARRAY: + return array(tape).at_pointer(json_pointer); + default: { + if (!json_pointer.empty()) { // a non-empty string can be invalid, or accessing a primitive (issue 2154) + if (is_pointer_well_formed(json_pointer)) { + return NO_SUCH_FIELD; + } + return INVALID_JSON_POINTER; + } + // an empty string means that we return the current node + dom::element copy(*this); + return simdjson_result(std::move(copy)); + } + } +} + +inline simdjson_result> element::at_path_with_wildcard(std::string_view json_path) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + + switch (tape.tape_ref_type()) { + case internal::tape_type::START_OBJECT: + return object(tape).at_path_with_wildcard(json_path); + case internal::tape_type::START_ARRAY: + return array(tape).at_path_with_wildcard(json_path); + default: + return std::vector{}; + } +} + +inline simdjson_result element::at_path(std::string_view json_path) const noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { return INVALID_JSON_POINTER; } + return at_pointer(json_pointer); +} +#ifndef SIMDJSON_DISABLE_DEPRECATED_API +[[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] +inline simdjson_result element::at(std::string_view json_pointer) const noexcept { + // version 0.4 of simdjson allowed non-compliant pointers + auto std_pointer = (json_pointer.empty() ? "" : "/") + std::string(json_pointer.begin(), json_pointer.end()); + return at_pointer(std_pointer); +} +#endif // SIMDJSON_DISABLE_DEPRECATED_API + +inline simdjson_result element::at(size_t index) const noexcept { + return get().at(index); +} +inline simdjson_result element::at_key(std::string_view key) const noexcept { + return get().at_key(key); +} +inline simdjson_result element::at_key_case_insensitive(std::string_view key) const noexcept { + return get().at_key_case_insensitive(key); +} +inline bool element::operator<(const element &other) const noexcept { + return tape.json_index < other.tape.json_index; +} +inline bool element::operator==(const element &other) const noexcept { + return tape.json_index == other.tape.json_index; +} + +inline bool element::dump_raw_tape(std::ostream &out) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + return tape.doc->dump_raw_tape(out); +} + + +inline std::ostream& operator<<(std::ostream& out, element_type type) { + switch (type) { + case element_type::ARRAY: + return out << "array"; + case element_type::OBJECT: + return out << "object"; + case element_type::INT64: + return out << "int64_t"; + case element_type::UINT64: + return out << "uint64_t"; + case element_type::DOUBLE: + return out << "double"; + case element_type::STRING: + return out << "string"; + case element_type::BOOL: + return out << "bool"; + case element_type::NULL_VALUE: + return out << "null"; + case element_type::BIGINT: + return out << "bigint"; + default: + return out << "unexpected content!!!"; // abort() usage is forbidden in the library + } +} + +} // namespace dom + +} // namespace simdjson + +#endif // SIMDJSON_ELEMENT_INL_H +/* end file simdjson/dom/element-inl.h */ + +#if SIMDJSON_SUPPORTS_RANGES +static_assert(std::ranges::view); +static_assert(std::ranges::sized_range); +#if SIMDJSON_EXCEPTIONS +static_assert(std::ranges::view>); +static_assert(std::ranges::sized_range>); +#endif // SIMDJSON_EXCEPTIONS +#endif // SIMDJSON_SUPPORTS_RANGES + +#endif // SIMDJSON_ARRAY_INL_H +/* end file simdjson/dom/array-inl.h */ +/* including simdjson/dom/document_stream-inl.h: #include "simdjson/dom/document_stream-inl.h" */ +/* begin file simdjson/dom/document_stream-inl.h */ +#ifndef SIMDJSON_DOCUMENT_STREAM_INL_H +#define SIMDJSON_DOCUMENT_STREAM_INL_H + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* skipped duplicate #include "simdjson/dom/document_stream.h" */ +/* skipped duplicate #include "simdjson/dom/element-inl.h" */ +/* including simdjson/dom/parser-inl.h: #include "simdjson/dom/parser-inl.h" */ +/* begin file simdjson/dom/parser-inl.h */ +#ifndef SIMDJSON_PARSER_INL_H +#define SIMDJSON_PARSER_INL_H + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* skipped duplicate #include "simdjson/dom/document_stream.h" */ +/* skipped duplicate #include "simdjson/implementation.h" */ +/* skipped duplicate #include "simdjson/internal/dom_parser_implementation.h" */ + +/* skipped duplicate #include "simdjson/error-inl.h" */ +/* skipped duplicate #include "simdjson/padded_string-inl.h" */ +/* skipped duplicate #include "simdjson/dom/document_stream-inl.h" */ +/* skipped duplicate #include "simdjson/dom/element-inl.h" */ + +#include +#include /* memcmp */ + +namespace simdjson { +namespace dom { + +// +// parser inline implementation +// +simdjson_inline parser::parser(size_t max_capacity) noexcept + : _max_capacity{max_capacity}, + loaded_bytes(nullptr) { +} +simdjson_inline parser::parser(parser &&other) noexcept = default; +simdjson_inline parser &parser::operator=(parser &&other) noexcept = default; + +inline bool parser::is_valid() const noexcept { return valid; } +inline int parser::get_error_code() const noexcept { return error; } +inline std::string parser::get_error_message() const noexcept { return error_message(error); } + +inline bool parser::dump_raw_tape(std::ostream &os) const noexcept { + return valid ? doc.dump_raw_tape(os) : false; +} + +inline simdjson_result parser::read_file(std::string_view path) noexcept { + const std::string path_copy(path); + // Open the file + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe + std::FILE *fp = std::fopen(path_copy.c_str(), "rb"); + SIMDJSON_POP_DISABLE_WARNINGS + + if (fp == nullptr) { + return IO_ERROR; + } + + // Get the file size + int ret; +#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS + ret = _fseeki64(fp, 0, SEEK_END); +#else + ret = std::fseek(fp, 0, SEEK_END); +#endif // _WIN64 + if(ret < 0) { + std::fclose(fp); + return IO_ERROR; + } +#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS + __int64 len = _ftelli64(fp); + if(len == -1L) { + std::fclose(fp); + return IO_ERROR; + } +#else + long len = std::ftell(fp); + if((len < 0) || (len == LONG_MAX)) { + std::fclose(fp); + return IO_ERROR; + } +#endif + + // Make sure we have enough capacity to load the file + if (_loaded_bytes_capacity < size_t(len)) { + loaded_bytes.reset( internal::allocate_padded_buffer(len) ); + if (!loaded_bytes) { + std::fclose(fp); + return MEMALLOC; + } + _loaded_bytes_capacity = len; + } + + // Read the string + std::rewind(fp); + size_t bytes_read = std::fread(loaded_bytes.get(), 1, len, fp); + if (std::fclose(fp) != 0 || bytes_read != size_t(len)) { + return IO_ERROR; + } + + return bytes_read; +} + +inline simdjson_result parser::load(std::string_view path) & noexcept { + return load_into_document(doc, path); +} + +inline simdjson_result parser::load_into_document(document& provided_doc, std::string_view path) & noexcept { + size_t len; + auto _error = read_file(path).get(len); + if (_error) { return _error; } + return parse_into_document(provided_doc, loaded_bytes.get(), len, false); +} + +inline simdjson_result parser::load_many(std::string_view path, size_t batch_size) noexcept { + size_t len; + auto _error = read_file(path).get(len); + if (_error) { return _error; } + if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + return document_stream(*this, reinterpret_cast(loaded_bytes.get()), len, batch_size); +} + +inline simdjson_result parser::parse_into_document(document& provided_doc, const uint8_t *buf, size_t len, bool realloc_if_needed) & noexcept { + // Important: we need to ensure that document has enough capacity. + // Important: It is possible that provided_doc is actually the internal 'doc' within the parser!!! + error_code _error = ensure_capacity(provided_doc, len); + if (_error) { return _error; } + if (realloc_if_needed) { + // Make sure we have enough capacity to copy len bytes + if (!loaded_bytes || _loaded_bytes_capacity < len) { + loaded_bytes.reset( internal::allocate_padded_buffer(len) ); + if (!loaded_bytes) { + return MEMALLOC; + } + _loaded_bytes_capacity = len; + } + std::memcpy(static_cast(loaded_bytes.get()), buf, len); + buf = reinterpret_cast(loaded_bytes.get()); + } + + if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { + buf += 3; + len -= 3; + } + implementation->_number_as_string = _number_as_string; + _error = implementation->parse(buf, len, provided_doc); + + if (_error) { return _error; } + + return provided_doc.root(); +} + +simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const char *buf, size_t len, bool realloc_if_needed) & noexcept { + return parse_into_document(provided_doc, reinterpret_cast(buf), len, realloc_if_needed); +} +simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const std::string &s) & noexcept { + return parse_into_document(provided_doc, s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); +} +simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const padded_string &s) & noexcept { + return parse_into_document(provided_doc, s.data(), s.length(), false); +} + + +inline simdjson_result parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) & noexcept { + return parse_into_document(doc, buf, len, realloc_if_needed); +} + +simdjson_inline simdjson_result parser::parse(const char *buf, size_t len, bool realloc_if_needed) & noexcept { + return parse(reinterpret_cast(buf), len, realloc_if_needed); +} +simdjson_inline simdjson_result parser::parse(const std::string &s) & noexcept { + return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); +} +simdjson_inline simdjson_result parser::parse(const padded_string &s) & noexcept { + return parse(s.data(), s.length(), false); +} +simdjson_inline simdjson_result parser::parse(const padded_string_view &v) & noexcept { + return parse(v.data(), v.length(), false); +} + +inline simdjson_result parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept { + if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { + buf += 3; + len -= 3; + } + return document_stream(*this, buf, len, batch_size); +} +inline simdjson_result parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept { + return parse_many(reinterpret_cast(buf), len, batch_size); +} +inline simdjson_result parser::parse_many(const std::string &s, size_t batch_size) noexcept { + return parse_many(s.data(), s.length(), batch_size); +} +inline simdjson_result parser::parse_many(const padded_string &s, size_t batch_size) noexcept { + return parse_many(s.data(), s.length(), batch_size); +} + +simdjson_inline size_t parser::capacity() const noexcept { + return implementation ? implementation->capacity() : 0; +} +simdjson_inline size_t parser::max_capacity() const noexcept { + return _max_capacity; +} +simdjson_pure simdjson_inline size_t parser::max_depth() const noexcept { + return implementation ? implementation->max_depth() : DEFAULT_MAX_DEPTH; +} + +simdjson_warn_unused +inline error_code parser::allocate(size_t capacity, size_t max_depth) noexcept { + // + // Reallocate implementation if needed + // + error_code err; + if (implementation) { + err = implementation->allocate(capacity, max_depth); + } else { + err = simdjson::get_active_implementation()->create_dom_parser_implementation(capacity, max_depth, implementation); + } + if (err) { return err; } + return SUCCESS; +} + +#ifndef SIMDJSON_DISABLE_DEPRECATED_API +simdjson_warn_unused +inline bool parser::allocate_capacity(size_t capacity, size_t max_depth) noexcept { + return !allocate(capacity, max_depth); +} +#endif // SIMDJSON_DISABLE_DEPRECATED_API + +inline error_code parser::ensure_capacity(size_t desired_capacity) noexcept { + return ensure_capacity(doc, desired_capacity); +} + + +inline error_code parser::ensure_capacity(document& target_document, size_t desired_capacity) noexcept { + // 1. It is wasteful to allocate a document and a parser for documents spanning less than MINIMAL_DOCUMENT_CAPACITY bytes. + // 2. If we allow desired_capacity = 0 then it is possible to exit this function with implementation == nullptr. + if(desired_capacity < MINIMAL_DOCUMENT_CAPACITY) { desired_capacity = MINIMAL_DOCUMENT_CAPACITY; } + // If we don't have enough capacity, (try to) automatically bump it. + // If the document needs allocation, do it too. + // Both in one if statement to minimize unlikely branching. + // + // Note: we must make sure that this function is called if capacity() == 0. We do so because we + // ensure that desired_capacity > 0. + if (simdjson_unlikely(capacity() < desired_capacity || target_document.capacity() < desired_capacity)) { + if (desired_capacity > max_capacity()) { + return error = CAPACITY; + } + error_code err1 = target_document.capacity() < desired_capacity ? target_document.allocate(desired_capacity) : SUCCESS; + error_code err2 = capacity() < desired_capacity ? allocate(desired_capacity, max_depth()) : SUCCESS; + if(err1 != SUCCESS) { return error = err1; } + if(err2 != SUCCESS) { return error = err2; } + } + return SUCCESS; +} + +simdjson_inline void parser::set_max_capacity(size_t max_capacity) noexcept { + if(max_capacity > MINIMAL_DOCUMENT_CAPACITY) { + _max_capacity = max_capacity; + } else { + _max_capacity = MINIMAL_DOCUMENT_CAPACITY; + } +} + +} // namespace dom +} // namespace simdjson + +#endif // SIMDJSON_PARSER_INL_H +/* end file simdjson/dom/parser-inl.h */ +/* skipped duplicate #include "simdjson/error-inl.h" */ +/* skipped duplicate #include "simdjson/internal/dom_parser_implementation.h" */ + +namespace simdjson { +namespace dom { + +#ifdef SIMDJSON_THREADS_ENABLED + +inline void stage1_worker::finish() { + // After calling "run" someone would call finish() to wait + // for the end of the processing. + // This function will wait until either the thread has done + // the processing or, else, the destructor has been called. + std::unique_lock lock(locking_mutex); + cond_var.wait(lock, [this]{return has_work == false;}); +} + +inline stage1_worker::~stage1_worker() { + // The thread may never outlive the stage1_worker instance + // and will always be stopped/joined before the stage1_worker + // instance is gone. + stop_thread(); +} + +inline void stage1_worker::start_thread() { + std::unique_lock lock(locking_mutex); + if(thread.joinable()) { + return; // This should never happen but we never want to create more than one thread. + } + thread = std::thread([this]{ + while(true) { + std::unique_lock thread_lock(locking_mutex); + // We wait for either "run" or "stop_thread" to be called. + cond_var.wait(thread_lock, [this]{return has_work || !can_work;}); + // If, for some reason, the stop_thread() method was called (i.e., the + // destructor of stage1_worker is called, then we want to immediately destroy + // the thread (and not do any more processing). + if(!can_work) { + break; + } + this->owner->stage1_thread_error = this->owner->run_stage1(*this->stage1_thread_parser, + this->_next_batch_start); + this->has_work = false; + // The condition variable call should be moved after thread_lock.unlock() for performance + // reasons but thread sanitizers may report it as a data race if we do. + // See https://stackoverflow.com/questions/35775501/c-should-condition-variable-be-notified-under-lock + cond_var.notify_one(); // will notify "finish" + thread_lock.unlock(); + } + } + ); +} + + +inline void stage1_worker::stop_thread() { + std::unique_lock lock(locking_mutex); + // We have to make sure that all locks can be released. + can_work = false; + has_work = false; + cond_var.notify_all(); + lock.unlock(); + if(thread.joinable()) { + thread.join(); + } +} + +inline void stage1_worker::run(document_stream * ds, dom::parser * stage1, size_t next_batch_start) { + std::unique_lock lock(locking_mutex); + owner = ds; + _next_batch_start = next_batch_start; + stage1_thread_parser = stage1; + has_work = true; + // The condition variable call should be moved after thread_lock.unlock() for performance + // reasons but thread sanitizers may report it as a data race if we do. + // See https://stackoverflow.com/questions/35775501/c-should-condition-variable-be-notified-under-lock + cond_var.notify_one(); // will notify the thread lock that we have work + lock.unlock(); +} +#endif + +simdjson_inline document_stream::document_stream( + dom::parser &_parser, + const uint8_t *_buf, + size_t _len, + size_t _batch_size +) noexcept + : parser{&_parser}, + buf{_buf}, + len{_len}, + batch_size{_batch_size <= MINIMAL_BATCH_SIZE ? MINIMAL_BATCH_SIZE : _batch_size}, + error{SUCCESS} +#ifdef SIMDJSON_THREADS_ENABLED + , use_thread(_parser.threaded) // we need to make a copy because _parser.threaded can change +#endif +{ +#ifdef SIMDJSON_THREADS_ENABLED + if(worker.get() == nullptr) { + error = MEMALLOC; + } +#endif +} + +simdjson_inline document_stream::document_stream() noexcept + : parser{nullptr}, + buf{nullptr}, + len{0}, + batch_size{0}, + error{UNINITIALIZED} +#ifdef SIMDJSON_THREADS_ENABLED + , use_thread(false) +#endif +{ +} + +simdjson_inline document_stream::~document_stream() noexcept { +#ifdef SIMDJSON_THREADS_ENABLED + worker.reset(); +#endif +} + +simdjson_inline document_stream::iterator::iterator() noexcept + : stream{nullptr}, finished{true} { +} + +simdjson_inline document_stream::iterator document_stream::begin() noexcept { + start(); + // If there are no documents, we're finished. + return iterator(this, error == EMPTY); +} + +simdjson_inline document_stream::iterator document_stream::end() noexcept { + return iterator(this, true); +} + +simdjson_inline document_stream::iterator::iterator(document_stream* _stream, bool is_end) noexcept + : stream{_stream}, finished{is_end} { +} + +simdjson_inline document_stream::iterator::reference document_stream::iterator::operator*() noexcept { + // Note that in case of error, we do not yet mark + // the iterator as "finished": this detection is done + // in the operator++ function since it is possible + // to call operator++ repeatedly while omitting + // calls to operator*. + if (stream->error) { return stream->error; } + return stream->parser->doc.root(); +} + +simdjson_inline document_stream::iterator& document_stream::iterator::operator++() noexcept { + // If there is an error, then we want the iterator + // to be finished, no matter what. (E.g., we do not + // keep generating documents with errors, or go beyond + // a document with errors.) + // + // Users do not have to call "operator*()" when they use operator++, + // so we need to end the stream in the operator++ function. + // + // Note that setting finished = true is essential otherwise + // we would enter an infinite loop. + if (stream->error) { finished = true; } + // Note that stream->error() is guarded against error conditions + // (it will immediately return if stream->error casts to false). + // In effect, this next function does nothing when (stream->error) + // is true (hence the risk of an infinite loop). + stream->next(); + // If that was the last document, we're finished. + // It is the only type of error we do not want to appear + // in operator*. + if (stream->error == EMPTY) { finished = true; } + // If we had any other kind of error (not EMPTY) then we want + // to pass it along to the operator* and we cannot mark the result + // as "finished" just yet. + return *this; +} + +simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { + return finished != other.finished; +} + +inline void document_stream::start() noexcept { + if (error) { return; } + error = parser->ensure_capacity(batch_size); + if (error) { return; } + // Always run the first stage 1 parse immediately + batch_start = 0; + error = run_stage1(*parser, batch_start); + while(error == EMPTY) { + // In exceptional cases, we may start with an empty block + batch_start = next_batch_start(); + if (batch_start >= len) { return; } + error = run_stage1(*parser, batch_start); + } + if (error) { return; } +#ifdef SIMDJSON_THREADS_ENABLED + if (use_thread && next_batch_start() < len) { + // Kick off the first thread if needed + error = stage1_thread_parser.ensure_capacity(batch_size); + if (error) { return; } + worker->start_thread(); + start_stage1_thread(); + if (error) { return; } + } +#endif // SIMDJSON_THREADS_ENABLED + next(); +} + +simdjson_inline size_t document_stream::iterator::current_index() const noexcept { + return stream->doc_index; +} + +simdjson_inline std::string_view document_stream::iterator::source() const noexcept { + const char* start = reinterpret_cast(stream->buf) + current_index(); + bool object_or_array = ((*start == '[') || (*start == '{')); + if(object_or_array) { + size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index - 1]; + return std::string_view(start, next_doc_index - current_index() + 1); + } else { + size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index]; + size_t svlen = next_doc_index - current_index(); + while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0')) { + svlen--; + } + return std::string_view(start, svlen); + } +} + + +inline void document_stream::next() noexcept { + // We always exit at once, once in an error condition. + if (error) { return; } + + // Load the next document from the batch + doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index]; + error = parser->implementation->stage2_next(parser->doc); + // If that was the last document in the batch, load another batch (if available) + while (error == EMPTY) { + batch_start = next_batch_start(); + if (batch_start >= len) { break; } + +#ifdef SIMDJSON_THREADS_ENABLED + if(use_thread) { + load_from_stage1_thread(); + } else { + error = run_stage1(*parser, batch_start); + } +#else + error = run_stage1(*parser, batch_start); +#endif + if (error) { continue; } // If the error was EMPTY, we may want to load another batch. + // Run stage 2 on the first document in the batch + doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index]; + error = parser->implementation->stage2_next(parser->doc); + } +} +inline size_t document_stream::size_in_bytes() const noexcept { + return len; +} + +inline size_t document_stream::truncated_bytes() const noexcept { + if(error == CAPACITY) { return len - batch_start; } + return parser->implementation->structural_indexes[parser->implementation->n_structural_indexes] - parser->implementation->structural_indexes[parser->implementation->n_structural_indexes + 1]; +} + +inline size_t document_stream::next_batch_start() const noexcept { + return batch_start + parser->implementation->structural_indexes[parser->implementation->n_structural_indexes]; +} + +inline error_code document_stream::run_stage1(dom::parser &p, size_t _batch_start) noexcept { + size_t remaining = len - _batch_start; + if (remaining <= batch_size) { + return p.implementation->stage1(&buf[_batch_start], remaining, stage1_mode::streaming_final); + } else { + return p.implementation->stage1(&buf[_batch_start], batch_size, stage1_mode::streaming_partial); + } +} + +#ifdef SIMDJSON_THREADS_ENABLED + +inline void document_stream::load_from_stage1_thread() noexcept { + worker->finish(); + // Swap to the parser that was loaded up in the thread. Make sure the parser has + // enough memory to swap to, as well. + std::swap(*parser, stage1_thread_parser); + error = stage1_thread_error; + if (error) { return; } + + // If there's anything left, start the stage 1 thread! + if (next_batch_start() < len) { + start_stage1_thread(); + } +} + +inline void document_stream::start_stage1_thread() noexcept { + // we call the thread on a lambda that will update + // this->stage1_thread_error + // there is only one thread that may write to this value + // TODO this is NOT exception-safe. + this->stage1_thread_error = UNINITIALIZED; // In case something goes wrong, make sure it's an error + size_t _next_batch_start = this->next_batch_start(); + + worker->run(this, & this->stage1_thread_parser, _next_batch_start); +} + +#endif // SIMDJSON_THREADS_ENABLED + +} // namespace dom + +simdjson_inline simdjson_result::simdjson_result() noexcept + : simdjson_result_base() { +} +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : simdjson_result_base(error) { +} +simdjson_inline simdjson_result::simdjson_result(dom::document_stream &&value) noexcept + : simdjson_result_base(std::forward(value)) { +} + +#if SIMDJSON_EXCEPTIONS +simdjson_inline dom::document_stream::iterator simdjson_result::begin() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.begin(); +} +simdjson_inline dom::document_stream::iterator simdjson_result::end() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.end(); +} +#else // SIMDJSON_EXCEPTIONS +#ifndef SIMDJSON_DISABLE_DEPRECATED_API +simdjson_inline dom::document_stream::iterator simdjson_result::begin() noexcept { + first.error = error(); + return first.begin(); +} +simdjson_inline dom::document_stream::iterator simdjson_result::end() noexcept { + first.error = error(); + return first.end(); +} +#endif // SIMDJSON_DISABLE_DEPRECATED_API +#endif // SIMDJSON_EXCEPTIONS + +} // namespace simdjson +#endif // SIMDJSON_DOCUMENT_STREAM_INL_H +/* end file simdjson/dom/document_stream-inl.h */ +/* including simdjson/dom/document-inl.h: #include "simdjson/dom/document-inl.h" */ +/* begin file simdjson/dom/document-inl.h */ +#ifndef SIMDJSON_DOCUMENT_INL_H +#define SIMDJSON_DOCUMENT_INL_H + +// Inline implementations go in here. + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* skipped duplicate #include "simdjson/dom/document.h" */ +/* skipped duplicate #include "simdjson/dom/element-inl.h" */ +/* skipped duplicate #include "simdjson/internal/tape_ref-inl.h" */ +/* including simdjson/internal/jsonformatutils.h: #include "simdjson/internal/jsonformatutils.h" */ +/* begin file simdjson/internal/jsonformatutils.h */ +#ifndef SIMDJSON_INTERNAL_JSONFORMATUTILS_H +#define SIMDJSON_INTERNAL_JSONFORMATUTILS_H + +/* skipped duplicate #include "simdjson/base.h" */ +#include +#include +#include + +namespace simdjson { +namespace internal { + +inline std::ostream& operator<<(std::ostream& out, const escape_json_string &str); + +class escape_json_string { +public: + escape_json_string(std::string_view _str) noexcept : str{_str} {} + operator std::string() const noexcept { std::stringstream s; s << *this; return s.str(); } +private: + std::string_view str; + friend std::ostream& operator<<(std::ostream& out, const escape_json_string &unescaped); +}; + +inline std::ostream& operator<<(std::ostream& out, const escape_json_string &unescaped) { + for (size_t i=0; i(unescaped.str[i]) <= 0x1F) { + // TODO can this be done once at the beginning, or will it mess up << char? + std::ios::fmtflags f(out.flags()); + out << "\\u" << std::hex << std::setw(4) << std::setfill('0') << int(unescaped.str[i]); + out.flags(f); + } else { + out << unescaped.str[i]; + } + } + } + return out; +} + +} // namespace internal +} // namespace simdjson + +#endif // SIMDJSON_INTERNAL_JSONFORMATUTILS_H +/* end file simdjson/internal/jsonformatutils.h */ + +#include + +namespace simdjson { +namespace dom { + +// +// document inline implementation +// +inline element document::root() const noexcept { + return element(internal::tape_ref(this, 1)); +} +simdjson_warn_unused +inline size_t document::capacity() const noexcept { + return allocated_capacity; +} + +simdjson_warn_unused +inline error_code document::allocate(size_t capacity) noexcept { + if (capacity == 0) { + string_buf.reset(); + tape.reset(); + allocated_capacity = 0; + return SUCCESS; + } + + // a pathological input like "[[[[..." would generate capacity tape elements, so + // need a capacity of at least capacity + 1, but it is also possible to do + // worse with "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6" + //where capacity + 1 tape elements are + // generated, see issue https://github.com/simdjson/simdjson/issues/345 + size_t tape_capacity = SIMDJSON_ROUNDUP_N(capacity + 3, 64); + // a document with only zero-length strings... could have capacity/3 string + // and we would need capacity/3 * 5 bytes on the string buffer + size_t string_capacity = SIMDJSON_ROUNDUP_N(5 * capacity / 3 + SIMDJSON_PADDING, 64); + string_buf.reset( new (std::nothrow) uint8_t[string_capacity]); + tape.reset(new (std::nothrow) uint64_t[tape_capacity]); + if(!(string_buf && tape)) { + allocated_capacity = 0; + string_buf.reset(); + tape.reset(); + return MEMALLOC; + } + // Technically the allocated_capacity might be larger than capacity + // so the next line is pessimistic. + allocated_capacity = capacity; + return SUCCESS; +} + +inline bool document::dump_raw_tape(std::ostream &os) const noexcept { + uint32_t string_length; + size_t tape_idx = 0; + uint64_t tape_val = tape[tape_idx]; + uint8_t type = uint8_t(tape_val >> 56); + os << tape_idx << " : " << type; + tape_idx++; + size_t how_many = 0; + if (type == 'r') { + how_many = size_t(tape_val & internal::JSON_VALUE_MASK); + } else { + // Error: no starting root node? + return false; + } + os << "\t// pointing to " << how_many << " (right after last node)\n"; + uint64_t payload; + for (; tape_idx < how_many; tape_idx++) { + os << tape_idx << " : "; + tape_val = tape[tape_idx]; + payload = tape_val & internal::JSON_VALUE_MASK; + type = uint8_t(tape_val >> 56); + switch (type) { + case '"': // we have a string + os << "string \""; + std::memcpy(&string_length, string_buf.get() + payload, sizeof(uint32_t)); + os << internal::escape_json_string(std::string_view( + reinterpret_cast(string_buf.get() + payload + sizeof(uint32_t)), + string_length + )); + os << '"'; + os << '\n'; + break; + case 'l': // we have a long int + if (tape_idx + 1 >= how_many) { + return false; + } + os << "integer " << static_cast(tape[++tape_idx]) << "\n"; + break; + case 'u': // we have a long uint + if (tape_idx + 1 >= how_many) { + return false; + } + os << "unsigned integer " << tape[++tape_idx] << "\n"; + break; + case 'd': // we have a double + os << "float "; + if (tape_idx + 1 >= how_many) { + return false; + } + double answer; + std::memcpy(&answer, &tape[++tape_idx], sizeof(answer)); + os << answer << '\n'; + break; + case 'n': // we have a null + os << "null\n"; + break; + case 't': // we have a true + os << "true\n"; + break; + case 'f': // we have a false + os << "false\n"; + break; + case '{': // we have an object + os << "{\t// pointing to next tape location " << uint32_t(payload) + << " (first node after the scope), " + << " saturated count " + << ((payload >> 32) & internal::JSON_COUNT_MASK)<< "\n"; + break; case '}': // we end an object + os << "}\t// pointing to previous tape location " << uint32_t(payload) + << " (start of the scope)\n"; + break; + case '[': // we start an array + os << "[\t// pointing to next tape location " << uint32_t(payload) + << " (first node after the scope), " + << " saturated count " + << ((payload >> 32) & internal::JSON_COUNT_MASK)<< "\n"; + break; + case ']': // we end an array + os << "]\t// pointing to previous tape location " << uint32_t(payload) + << " (start of the scope)\n"; + break; + case 'r': // we start and end with the root node + // should we be hitting the root node? + return false; + case 'Z': // we have a big integer + os << "bigint "; + std::memcpy(&string_length, string_buf.get() + payload, sizeof(uint32_t)); + os << std::string_view( + reinterpret_cast(string_buf.get() + payload + sizeof(uint32_t)), + string_length + ); + os << '\n'; + break; + default: + return false; + } + } + tape_val = tape[tape_idx]; + payload = tape_val & internal::JSON_VALUE_MASK; + type = uint8_t(tape_val >> 56); + os << tape_idx << " : " << type << "\t// pointing to " << payload + << " (start root)\n"; + return true; +} + +} // namespace dom +} // namespace simdjson + +#endif // SIMDJSON_DOCUMENT_INL_H +/* end file simdjson/dom/document-inl.h */ +/* skipped duplicate #include "simdjson/dom/element-inl.h" */ +/* skipped duplicate #include "simdjson/dom/object-inl.h" */ +/* skipped duplicate #include "simdjson/dom/parser-inl.h" */ +/* skipped duplicate #include "simdjson/internal/tape_ref-inl.h" */ +/* including simdjson/dom/serialization-inl.h: #include "simdjson/dom/serialization-inl.h" */ +/* begin file simdjson/dom/serialization-inl.h */ + +#ifndef SIMDJSON_SERIALIZATION_INL_H +#define SIMDJSON_SERIALIZATION_INL_H + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* skipped duplicate #include "simdjson/dom/parser.h" */ +/* skipped duplicate #include "simdjson/dom/serialization.h" */ +/* skipped duplicate #include "simdjson/internal/tape_type.h" */ + +/* skipped duplicate #include "simdjson/dom/array-inl.h" */ +/* skipped duplicate #include "simdjson/dom/object-inl.h" */ +/* skipped duplicate #include "simdjson/internal/tape_ref-inl.h" */ + +#include + +namespace simdjson { +namespace dom { +inline bool parser::print_json(std::ostream &os) const noexcept { + if (!valid) { + return false; + } + simdjson::internal::string_builder<> sb; + sb.append(doc.root()); + std::string_view answer = sb.str(); + os << answer; + return true; +} + +inline std::ostream &operator<<(std::ostream &out, + simdjson::dom::element value) { + simdjson::internal::string_builder<> sb; + sb.append(value); + return (out << sb.str()); +} +#if SIMDJSON_EXCEPTIONS +inline std::ostream & +operator<<(std::ostream &out, + simdjson::simdjson_result x) { + if (x.error()) { + throw simdjson::simdjson_error(x.error()); + } + return (out << x.value()); +} +#endif +inline std::ostream &operator<<(std::ostream &out, simdjson::dom::array value) { + simdjson::internal::string_builder<> sb; + sb.append(value); + return (out << sb.str()); +} +#if SIMDJSON_EXCEPTIONS +inline std::ostream & +operator<<(std::ostream &out, + simdjson::simdjson_result x) { + if (x.error()) { + throw simdjson::simdjson_error(x.error()); + } + return (out << x.value()); +} +#endif +inline std::ostream &operator<<(std::ostream &out, + simdjson::dom::object value) { + simdjson::internal::string_builder<> sb; + sb.append(value); + return (out << sb.str()); +} +#if SIMDJSON_EXCEPTIONS +inline std::ostream & +operator<<(std::ostream &out, + simdjson::simdjson_result x) { + if (x.error()) { + throw simdjson::simdjson_error(x.error()); + } + return (out << x.value()); +} +#endif + +} // namespace dom + +/*** + * Number utility functions + **/ +namespace { +/**@private + * Escape sequence like \b or \u0001 + * We expect that most compilers will use 8 bytes for this data structure. + **/ +struct escape_sequence { + uint8_t length; + const char + string[7]; // technically, we only ever need 6 characters, we pad to 8 +}; +/**@private + * This converts a signed integer into a character sequence. + * The caller is responsible for providing enough memory (at least + * 20 characters.) + * Though various runtime libraries provide itoa functions, + * it is not part of the C++ standard. The C++17 standard + * adds the to_chars functions which would do as well, but + * we want to support C++11. + */ +static char *fast_itoa(char *output, int64_t value) noexcept { + // This is a standard implementation of itoa. + char buffer[20]; + uint64_t value_positive; + // In general, negating a signed integer is unsafe. + if (value < 0) { + *output++ = '-'; + // Doing value_positive = -value; while avoiding + // undefined behavior warnings. + // It assumes two complement's which is universal at this + // point in time. + std::memcpy(&value_positive, &value, sizeof(value)); + value_positive = (~value_positive) + 1; // this is a negation + } else { + value_positive = value; + } + // We work solely with value_positive. It *might* be easier + // for an optimizing compiler to deal with an unsigned variable + // as far as performance goes. + const char *const end_buffer = buffer + 20; + char *write_pointer = buffer + 19; + // A faster approach is possible if we expect large integers: + // unroll the loop (work in 100s, 1000s) and use some kind of + // memoization. + while (value_positive >= 10) { + *write_pointer-- = char('0' + (value_positive % 10)); + value_positive /= 10; + } + *write_pointer = char('0' + value_positive); + size_t len = end_buffer - write_pointer; + std::memcpy(output, write_pointer, len); + return output + len; +} +/**@private + * This converts an unsigned integer into a character sequence. + * The caller is responsible for providing enough memory (at least + * 19 characters.) + * Though various runtime libraries provide itoa functions, + * it is not part of the C++ standard. The C++17 standard + * adds the to_chars functions which would do as well, but + * we want to support C++11. + */ +static char *fast_itoa(char *output, uint64_t value) noexcept { + // This is a standard implementation of itoa. + char buffer[20]; + const char *const end_buffer = buffer + 20; + char *write_pointer = buffer + 19; + // A faster approach is possible if we expect large integers: + // unroll the loop (work in 100s, 1000s) and use some kind of + // memoization. + while (value >= 10) { + *write_pointer-- = char('0' + (value % 10)); + value /= 10; + }; + *write_pointer = char('0' + value); + size_t len = end_buffer - write_pointer; + std::memcpy(output, write_pointer, len); + return output + len; +} + +} // anonymous namespace +namespace internal { + +/*** + * Minifier/formatter code. + **/ + +template +simdjson_inline void base_formatter::number(uint64_t x) { + char number_buffer[24]; + char *newp = fast_itoa(number_buffer, x); + chars(number_buffer, newp); +} + +template +simdjson_inline void base_formatter::number(int64_t x) { + char number_buffer[24]; + char *newp = fast_itoa(number_buffer, x); + chars(number_buffer, newp); +} + +template +simdjson_inline void base_formatter::number(double x) { + char number_buffer[24]; + // Currently, passing the nullptr to the second argument is + // safe because our implementation does not check the second + // argument. + char *newp = internal::to_chars(number_buffer, nullptr, x); + chars(number_buffer, newp); +} + +template +simdjson_inline void base_formatter::start_array() { + one_char('['); +} + +template +simdjson_inline void base_formatter::end_array() { + one_char(']'); +} + +template +simdjson_inline void base_formatter::start_object() { + one_char('{'); +} + +template +simdjson_inline void base_formatter::end_object() { + one_char('}'); +} + +template +simdjson_inline void base_formatter::comma() { + one_char(','); +} + +template +simdjson_inline void base_formatter::true_atom() { + const char *s = "true"; + chars(s, s + 4); +} + +template +simdjson_inline void base_formatter::false_atom() { + const char *s = "false"; + chars(s, s + 5); +} + +template +simdjson_inline void base_formatter::null_atom() { + const char *s = "null"; + chars(s, s + 4); +} + +template +simdjson_inline void base_formatter::one_char(char c) { + buffer.push_back(c); +} + +template +simdjson_inline void base_formatter::chars(const char *begin, + const char *end) { + buffer.append(begin, end); +} + +template +simdjson_inline void +base_formatter::key(std::string_view unescaped) { + string(unescaped); + one_char(':'); +} + +template +simdjson_inline void +base_formatter::string(std::string_view unescaped) { + one_char('\"'); + size_t i = 0; + // Fast path for the case where we have no control character, no ", and no + // backslash. This should include most keys. + // + // We would like to use 'bool' but some compilers take offense to bitwise + // operation with bool types. + constexpr static char needs_escaping[] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + for (; i + 8 <= unescaped.length(); i += 8) { + // Poor's man vectorization. This could get much faster if we used SIMD. + // + // It is not the case that replacing '|' with '||' would be neutral + // performance-wise. + if (needs_escaping[uint8_t(unescaped[i])] | + needs_escaping[uint8_t(unescaped[i + 1])] | + needs_escaping[uint8_t(unescaped[i + 2])] | + needs_escaping[uint8_t(unescaped[i + 3])] | + needs_escaping[uint8_t(unescaped[i + 4])] | + needs_escaping[uint8_t(unescaped[i + 5])] | + needs_escaping[uint8_t(unescaped[i + 6])] | + needs_escaping[uint8_t(unescaped[i + 7])]) { + break; + } + } + for (; i < unescaped.length(); i++) { + if (needs_escaping[uint8_t(unescaped[i])]) { + break; + } + } + // The following is also possible and omits a 256-byte table, but it is + // slower: for (; (i < unescaped.length()) && (uint8_t(unescaped[i]) > 0x1F) + // && (unescaped[i] != '\"') && (unescaped[i] != '\\'); i++) {} + + // At least for long strings, the following should be fast. We could + // do better by integrating the checks and the insertion. + chars(unescaped.data(), unescaped.data() + i); + // We caught a control character if we enter this loop (slow). + // Note that we are do not restart from the beginning, but rather we continue + // from the point where we encountered something that requires escaping. + for (; i < unescaped.length(); i++) { + switch (unescaped[i]) { + case '\"': { + const char *s = "\\\""; + chars(s, s + 2); + } break; + case '\\': { + const char *s = "\\\\"; + chars(s, s + 2); + } break; + default: + if (uint8_t(unescaped[i]) <= 0x1F) { + // If packed, this uses 8 * 32 bytes. + // Note that we expect most compilers to embed this code in the data + // section. + constexpr static escape_sequence escaped[32] = { + {6, "\\u0000"}, {6, "\\u0001"}, {6, "\\u0002"}, {6, "\\u0003"}, + {6, "\\u0004"}, {6, "\\u0005"}, {6, "\\u0006"}, {6, "\\u0007"}, + {2, "\\b"}, {2, "\\t"}, {2, "\\n"}, {6, "\\u000b"}, + {2, "\\f"}, {2, "\\r"}, {6, "\\u000e"}, {6, "\\u000f"}, + {6, "\\u0010"}, {6, "\\u0011"}, {6, "\\u0012"}, {6, "\\u0013"}, + {6, "\\u0014"}, {6, "\\u0015"}, {6, "\\u0016"}, {6, "\\u0017"}, + {6, "\\u0018"}, {6, "\\u0019"}, {6, "\\u001a"}, {6, "\\u001b"}, + {6, "\\u001c"}, {6, "\\u001d"}, {6, "\\u001e"}, {6, "\\u001f"}}; + auto u = escaped[uint8_t(unescaped[i])]; + chars(u.string, u.string + u.length); + } else { + one_char(unescaped[i]); + } + } // switch + } // for + one_char('\"'); +} + +template inline void base_formatter::clear() { + buffer.clear(); +} + +template +simdjson_inline std::string_view base_formatter::str() const { + return buffer.str(); +} + +simdjson_inline void mini_formatter::print_newline() { return; } + +simdjson_inline void mini_formatter::print_indents(size_t depth) { + (void)depth; + return; +} + +simdjson_inline void mini_formatter::print_space() { return; } + +simdjson_inline void pretty_formatter::print_newline() { one_char('\n'); } + +simdjson_inline void pretty_formatter::print_indents(size_t depth) { + if (this->indent_step <= 0) { + return; + } + for (size_t i = 0; i < this->indent_step * depth; i++) { + one_char(' '); + } +} + +simdjson_inline void pretty_formatter::print_space() { one_char(' '); } + +/*** + * String building code. + **/ + +template +inline void string_builder::append(simdjson::dom::element value) { + // using tape_type = simdjson::internal::tape_type; + size_t depth = 0; + constexpr size_t MAX_DEPTH = 16; + bool is_object[MAX_DEPTH]; + is_object[0] = false; + bool after_value = false; + + internal::tape_ref iter(value.tape); + do { + // print commas after each value + if (after_value) { + format.comma(); + format.print_newline(); + } + + format.print_indents(depth); + + // If we are in an object, print the next key and :, and skip to the next + // value. + if (is_object[depth]) { + format.key(iter.get_string_view()); + format.print_space(); + iter.json_index++; + } + switch (iter.tape_ref_type()) { + + // Arrays + case tape_type::START_ARRAY: { + // If we're too deep, we need to recurse to go deeper. + depth++; + if (simdjson_unlikely(depth >= MAX_DEPTH)) { + append(simdjson::dom::array(iter)); + iter.json_index = iter.matching_brace_index() - 1; // Jump to the ] + depth--; + break; + } + + // Output start [ + format.start_array(); + iter.json_index++; + + // Handle empty [] (we don't want to come back around and print commas) + if (iter.tape_ref_type() == tape_type::END_ARRAY) { + format.end_array(); + depth--; + break; + } + + is_object[depth] = false; + after_value = false; + format.print_newline(); + continue; + } + + // Objects + case tape_type::START_OBJECT: { + // If we're too deep, we need to recurse to go deeper. + depth++; + if (simdjson_unlikely(depth >= MAX_DEPTH)) { + append(simdjson::dom::object(iter)); + iter.json_index = iter.matching_brace_index() - 1; // Jump to the } + depth--; + break; + } + + // Output start { + format.start_object(); + iter.json_index++; + + // Handle empty {} (we don't want to come back around and print commas) + if (iter.tape_ref_type() == tape_type::END_OBJECT) { + format.end_object(); + depth--; + break; + } + + is_object[depth] = true; + after_value = false; + format.print_newline(); + continue; + } + + // Scalars + case tape_type::STRING: + format.string(iter.get_string_view()); + break; + case tape_type::BIGINT: { + // Big integer stored as string — output raw digits (no quotes) + auto sv = iter.get_string_view(); + format.chars(sv.data(), sv.data() + sv.size()); + break; + } + case tape_type::INT64: + format.number(iter.next_tape_value()); + iter.json_index++; // numbers take up 2 spots, so we need to increment + // extra + break; + case tape_type::UINT64: + format.number(iter.next_tape_value()); + iter.json_index++; // numbers take up 2 spots, so we need to increment + // extra + break; + case tape_type::DOUBLE: + format.number(iter.next_tape_value()); + iter.json_index++; // numbers take up 2 spots, so we need to increment + // extra + break; + case tape_type::TRUE_VALUE: + format.true_atom(); + break; + case tape_type::FALSE_VALUE: + format.false_atom(); + break; + case tape_type::NULL_VALUE: + format.null_atom(); + break; + + // These are impossible + case tape_type::END_ARRAY: + case tape_type::END_OBJECT: + case tape_type::ROOT: + SIMDJSON_UNREACHABLE(); + } + iter.json_index++; + after_value = true; + + // Handle multiple ends in a row + while (depth != 0 && (iter.tape_ref_type() == tape_type::END_ARRAY || + iter.tape_ref_type() == tape_type::END_OBJECT)) { + format.print_newline(); + depth--; + format.print_indents(depth); + if (iter.tape_ref_type() == tape_type::END_ARRAY) { + format.end_array(); + } else { + format.end_object(); + } + iter.json_index++; + } + + // Stop when we're at depth 0 + } while (depth != 0); + + format.print_newline(); +} + +template +inline void string_builder::append(simdjson::dom::object value) { + format.start_object(); + auto pair = value.begin(); + auto end = value.end(); + if (pair != end) { + append(*pair); + for (++pair; pair != end; ++pair) { + format.comma(); + append(*pair); + } + } + format.end_object(); +} + +template +inline void string_builder::append(simdjson::dom::array value) { + format.start_array(); + auto iter = value.begin(); + auto end = value.end(); + if (iter != end) { + append(*iter); + for (++iter; iter != end; ++iter) { + format.comma(); + append(*iter); + } + } + format.end_array(); +} + +template +simdjson_inline void +string_builder::append(simdjson::dom::key_value_pair kv) { + format.key(kv.key); + append(kv.value); +} + +template +simdjson_inline void string_builder::clear() { + format.clear(); +} + +template +simdjson_inline std::string_view string_builder::str() const { + return format.str(); +} + +} // namespace internal +} // namespace simdjson + +#endif +/* end file simdjson/dom/serialization-inl.h */ +/* including simdjson/dom/fractured_json-inl.h: #include "simdjson/dom/fractured_json-inl.h" */ +/* begin file simdjson/dom/fractured_json-inl.h */ +#ifndef SIMDJSON_DOM_FRACTURED_JSON_INL_H +#define SIMDJSON_DOM_FRACTURED_JSON_INL_H + +/* skipped duplicate #include "simdjson/dom/fractured_json.h" */ +/* skipped duplicate #include "simdjson/dom/serialization.h" */ +/* skipped duplicate #include "simdjson/dom/element-inl.h" */ +/* skipped duplicate #include "simdjson/dom/array-inl.h" */ +/* skipped duplicate #include "simdjson/dom/object-inl.h" */ +/* skipped duplicate #include "simdjson/dom/parser-inl.h" */ +/* skipped duplicate #include "simdjson/padded_string.h" */ +/* including simdjson/internal/json_structure_analyzer.h: #include "simdjson/internal/json_structure_analyzer.h" */ +/* begin file simdjson/internal/json_structure_analyzer.h */ +#ifndef SIMDJSON_INTERNAL_JSON_STRUCTURE_ANALYZER_H +#define SIMDJSON_INTERNAL_JSON_STRUCTURE_ANALYZER_H + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* skipped duplicate #include "simdjson/dom/element.h" */ +/* skipped duplicate #include "simdjson/dom/array.h" */ +/* skipped duplicate #include "simdjson/dom/object.h" */ +/* skipped duplicate #include "simdjson/dom/fractured_json.h" */ +/* skipped duplicate #include "simdjson/internal/tape_type.h" */ + +#include +#include +#include +#include + +namespace simdjson { +namespace internal { + +/** + * Layout mode for fractured JSON formatting. + */ +enum class layout_mode { + INLINE, // Single line: [1, 2, 3] or {"a": 1} + COMPACT_MULTILINE, // Multiple items per line with breaks + TABLE, // Tabular format for arrays of similar objects + EXPANDED // Traditional multi-line with indentation +}; + +/** + * Metrics computed for a JSON element during structure analysis. + * These metrics drive layout decisions and contain child metrics for recursive formatting. + */ +struct element_metrics { + /** Nesting depth score (0 = scalar, 1 = flat container, etc.) */ + size_t complexity = 0; + + /** Estimated character length if rendered inline (minified + spaces) */ + size_t estimated_inline_len = 0; + + /** Number of direct children (0 for scalars) */ + size_t child_count = 0; + + /** Pre-computed: can this element be rendered inline? */ + bool can_inline = false; + + /** Is this an array where all elements have similar structure? */ + bool is_uniform_array = false; + + /** For uniform arrays of objects: the common keys */ + std::vector common_keys{}; + + /** Recommended layout mode based on analysis */ + layout_mode recommended_layout = layout_mode::EXPANDED; + + /** Child metrics for arrays and objects (in order of iteration) */ + std::vector children{}; +}; + +/** + * Analyzes JSON structure to compute metrics for formatting decisions. + * + * The analyzer performs a single pass over the DOM to compute: + * - Complexity (nesting depth) + * - Estimated inline length + * - Array uniformity for table detection + * + * Metrics are stored hierarchically with child metrics embedded in parent metrics, + * enabling efficient lookup during formatting without address-based caching. + */ +class structure_analyzer { +public: + /** Default constructor */ + structure_analyzer() : current_opts_(nullptr) {} + + /** Copy constructor - deleted since class has pointer member */ + structure_analyzer(const structure_analyzer&) = delete; + + /** Copy assignment - deleted since class has pointer member */ + structure_analyzer& operator=(const structure_analyzer&) = delete; + + /** Move constructor */ + structure_analyzer(structure_analyzer&&) = default; + + /** Move assignment */ + structure_analyzer& operator=(structure_analyzer&&) = default; + + /** + * Analyze a DOM element and compute metrics. + * @param elem The element to analyze + * @param opts Formatting options that affect metric computation + * @return Metrics for the root element (with child metrics embedded) + */ + element_metrics analyze(const dom::element& elem, + const fractured_json_options& opts); + + /** + * Clear state. + */ + void clear(); + + /** + * Analyze an array element directly (for standalone array formatting). + * @param arr The array to analyze + * @param opts Formatting options + * @return Metrics for the array + */ + element_metrics analyze_array(const dom::array& arr, + const fractured_json_options& opts); + + /** + * Analyze an object element directly (for standalone object formatting). + * @param obj The object to analyze + * @param opts Formatting options + * @return Metrics for the object + */ + element_metrics analyze_object(const dom::object& obj, + const fractured_json_options& opts); + +private: + const fractured_json_options* current_opts_ = nullptr; + + /** Recursive analysis implementation */ + element_metrics analyze_element(const dom::element& elem, size_t depth); + + /** Analyze scalar values (strings, numbers, booleans, null) */ + element_metrics analyze_scalar(const dom::element& elem); + + /** Analyze an array element */ + element_metrics analyze_array(const dom::array& arr, size_t depth); + + /** Analyze an object element */ + element_metrics analyze_object(const dom::object& obj, size_t depth); + + /** Estimate inline length for a string (including quotes and escaping) */ + size_t estimate_string_length(std::string_view s) const; + + /** Estimate inline length for a number */ + size_t estimate_number_length(double d) const; + size_t estimate_number_length(int64_t i) const; + size_t estimate_number_length(uint64_t u) const; + + /** + * Check if an array contains uniform objects suitable for table formatting. + * @param arr The array to check + * @param common_keys Output: keys common to all objects + * @return true if the array is suitable for table formatting + */ + bool check_array_uniformity(const dom::array& arr, + std::vector& common_keys) const; + + /** + * Compute similarity between two objects. + * @return Fraction of keys that are common (0.0 to 1.0) + */ + double compute_object_similarity(const dom::object& a, + const dom::object& b) const; + + /** + * Decide the recommended layout mode based on metrics and options. + */ + layout_mode decide_layout(const element_metrics& metrics, + size_t depth, + size_t available_width) const; +}; + +} // namespace internal +} // namespace simdjson + +#endif // SIMDJSON_INTERNAL_JSON_STRUCTURE_ANALYZER_H +/* end file simdjson/internal/json_structure_analyzer.h */ +/* including simdjson/internal/fractured_formatter.h: #include "simdjson/internal/fractured_formatter.h" */ +/* begin file simdjson/internal/fractured_formatter.h */ +#ifndef SIMDJSON_INTERNAL_FRACTURED_FORMATTER_H +#define SIMDJSON_INTERNAL_FRACTURED_FORMATTER_H + +/* skipped duplicate #include "simdjson/dom/serialization.h" */ +/* skipped duplicate #include "simdjson/dom/fractured_json.h" */ +/* skipped duplicate #include "simdjson/internal/json_structure_analyzer.h" */ + +namespace simdjson { +namespace internal { + +/** + * Fractured JSON formatter using CRTP pattern. + * + * This formatter intelligently chooses between different layout modes + * (inline, compact multiline, table, expanded) based on pre-computed + * structure metrics. + */ +class fractured_formatter : public base_formatter { +public: + explicit fractured_formatter(const fractured_json_options& opts = {}); + + /** CRTP hook: print newline (context-aware) */ + simdjson_inline void print_newline(); + + /** CRTP hook: print indentation */ + simdjson_inline void print_indents(size_t depth); + + /** CRTP hook: print space (context-aware) */ + simdjson_inline void print_space(); + + /** Set the current layout mode */ + void set_layout_mode(layout_mode mode); + + /** Get the current layout mode */ + layout_mode get_layout_mode() const; + + /** Set current depth for formatting decisions */ + void set_depth(size_t depth); + + /** Get current depth */ + size_t get_depth() const; + + /** Track current line length for compact multiline decisions */ + void track_line_length(size_t chars); + + /** Reset line length (after newline) */ + void reset_line_length(); + + /** Get current line length */ + size_t get_line_length() const; + + /** Check if we should break to a new line in compact mode */ + bool should_break_line(size_t upcoming_length) const; + + /** Get the options */ + const fractured_json_options& options() const; + + // Table formatting support + /** Begin a table row */ + void begin_table_row(); + + /** End a table row */ + void end_table_row(); + + /** Set column widths for table alignment */ + void set_column_widths(const std::vector& widths); + + /** Get current column index in table mode */ + size_t get_column_index() const; + + /** Advance to next column */ + void next_column(); + + /** Add padding to align with column width */ + void align_to_column_width(size_t actual_width); + +private: + fractured_json_options options_; + layout_mode current_layout_ = layout_mode::EXPANDED; + size_t current_depth_ = 0; + size_t current_line_length_ = 0; + + // Table state + bool in_table_mode_ = false; + std::vector column_widths_; + size_t current_column_ = 0; +}; + +/** + * Specialized string builder for fractured JSON formatting. + * + * This builder performs two passes: + * 1. Analyze the structure to compute metrics + * 2. Format using the metrics to make layout decisions + */ +class fractured_string_builder { +public: + fractured_string_builder(const fractured_json_options& opts = {}); + + /** Append a DOM element with fractured formatting */ + void append(const dom::element& value); + + /** Append a DOM array with fractured formatting */ + void append(const dom::array& value); + + /** Append a DOM object with fractured formatting */ + void append(const dom::object& value); + + /** Clear the builder */ + simdjson_inline void clear(); + + /** Get the formatted string */ + simdjson_inline std::string_view str() const; + +private: + fractured_formatter format_; + structure_analyzer analyzer_; + fractured_json_options options_; + + /** Format an element using pre-computed metrics */ + void format_element(const dom::element& elem, const element_metrics& metrics, size_t depth); + + /** Format an array with the appropriate layout */ + void format_array(const dom::array& arr, const element_metrics& metrics, size_t depth); + + /** Format an array inline: [1, 2, 3] */ + void format_array_inline(const dom::array& arr, const element_metrics& metrics); + + /** Format an array with compact multiline: multiple items per line */ + void format_array_compact_multiline(const dom::array& arr, const element_metrics& metrics, size_t depth); + + /** Format an array as a table */ + void format_array_as_table(const dom::array& arr, const element_metrics& metrics, size_t depth); + + /** Format an array expanded: one item per line */ + void format_array_expanded(const dom::array& arr, const element_metrics& metrics, size_t depth); + + /** Format an object with the appropriate layout */ + void format_object(const dom::object& obj, const element_metrics& metrics, size_t depth); + + /** Format an object inline: {"a": 1, "b": 2} */ + void format_object_inline(const dom::object& obj, const element_metrics& metrics); + + /** Format an object expanded: one key per line */ + void format_object_expanded(const dom::object& obj, const element_metrics& metrics, size_t depth); + + /** Format a scalar value */ + void format_scalar(const dom::element& elem); + + /** Calculate column widths for table formatting */ + std::vector calculate_column_widths(const dom::array& arr, + const std::vector& columns) const; + + /** Measure the actual formatted length of a value (for alignment) */ + size_t measure_value_length(const dom::element& elem) const; +}; + +} // namespace internal +} // namespace simdjson + +#endif // SIMDJSON_INTERNAL_FRACTURED_FORMATTER_H +/* end file simdjson/internal/fractured_formatter.h */ + +#include +#include +#include + +namespace simdjson { +namespace internal { + +// +// Structure Analyzer Implementation +// + +inline element_metrics structure_analyzer::analyze(const dom::element& elem, + const fractured_json_options& opts) { + current_opts_ = &opts; + return analyze_element(elem, 0); +} + +inline void structure_analyzer::clear() { + current_opts_ = nullptr; +} + +inline element_metrics structure_analyzer::analyze_array(const dom::array& arr, + const fractured_json_options& opts) { + current_opts_ = &opts; + return analyze_array(arr, 0); +} + +inline element_metrics structure_analyzer::analyze_object(const dom::object& obj, + const fractured_json_options& opts) { + current_opts_ = &opts; + return analyze_object(obj, 0); +} + +inline element_metrics structure_analyzer::analyze_element(const dom::element& elem, size_t depth) { + switch (elem.type()) { + case dom::element_type::ARRAY: { + dom::array arr; + if (elem.get_array().get(arr) == SUCCESS) { + return analyze_array(arr, depth); + } + break; + } + case dom::element_type::OBJECT: { + dom::object obj; + if (elem.get_object().get(obj) == SUCCESS) { + return analyze_object(obj, depth); + } + break; + } + default: + // Handle all scalar types with a helper + return analyze_scalar(elem); + } + return element_metrics{}; +} + +inline element_metrics structure_analyzer::analyze_scalar(const dom::element& elem) { + element_metrics metrics; + metrics.complexity = 0; + metrics.child_count = 0; + metrics.can_inline = true; + metrics.recommended_layout = layout_mode::INLINE; + + switch (elem.type()) { + case dom::element_type::STRING: { + std::string_view str; + if (elem.get_string().get(str) == SUCCESS) { + metrics.estimated_inline_len = estimate_string_length(str); + } + break; + } + case dom::element_type::INT64: { + int64_t val; + if (elem.get_int64().get(val) == SUCCESS) { + metrics.estimated_inline_len = estimate_number_length(val); + } + break; + } + case dom::element_type::UINT64: { + uint64_t val; + if (elem.get_uint64().get(val) == SUCCESS) { + metrics.estimated_inline_len = estimate_number_length(val); + } + break; + } + case dom::element_type::DOUBLE: { + double val; + if (elem.get_double().get(val) == SUCCESS) { + metrics.estimated_inline_len = estimate_number_length(val); + } + break; + } + case dom::element_type::BOOL: { + bool val; + if (elem.get_bool().get(val) == SUCCESS) { + metrics.estimated_inline_len = val ? 4 : 5; // "true" or "false" + } + break; + } + case dom::element_type::NULL_VALUE: + metrics.estimated_inline_len = 4; // "null" + break; + default: + break; + } + + return metrics; +} + +inline element_metrics structure_analyzer::analyze_array(const dom::array& arr, + size_t depth) { + element_metrics metrics; + metrics.complexity = 1; // At least 1 for being an array + metrics.estimated_inline_len = 2; // "[]" + metrics.child_count = 0; + + size_t max_child_complexity = 0; + bool first = true; + + for (dom::element child : arr) { + if (!first) { + metrics.estimated_inline_len += 2; // ", " + } + first = false; + + element_metrics child_metrics = analyze_element(child, depth + 1); + metrics.estimated_inline_len += child_metrics.estimated_inline_len; + max_child_complexity = (std::max)(max_child_complexity, child_metrics.complexity); + metrics.child_count++; + metrics.children.push_back(std::move(child_metrics)); + } + + // Complexity is 1 + max child complexity + metrics.complexity = 1 + max_child_complexity; + + // Check if can inline + metrics.can_inline = (metrics.complexity <= current_opts_->max_inline_complexity) && + (metrics.estimated_inline_len <= current_opts_->max_inline_length); + + // Check for uniform array (table formatting) + if (current_opts_->enable_table_format && + metrics.child_count >= current_opts_->min_table_rows) { + metrics.is_uniform_array = check_array_uniformity(arr, metrics.common_keys); + } + + // Decide layout + if (metrics.child_count == 0) { + metrics.recommended_layout = layout_mode::INLINE; + } else if (metrics.can_inline) { + metrics.recommended_layout = layout_mode::INLINE; + } else if (metrics.is_uniform_array && !metrics.common_keys.empty()) { + metrics.recommended_layout = layout_mode::TABLE; + } else if (current_opts_->enable_compact_multiline && + max_child_complexity <= current_opts_->max_compact_array_complexity) { + metrics.recommended_layout = layout_mode::COMPACT_MULTILINE; + } else { + metrics.recommended_layout = layout_mode::EXPANDED; + } + + return metrics; +} + +inline element_metrics structure_analyzer::analyze_object(const dom::object& obj, + size_t depth) { + element_metrics metrics; + metrics.complexity = 1; + metrics.estimated_inline_len = 2; // "{}" + metrics.child_count = 0; + + size_t max_child_complexity = 0; + bool first = true; + + for (dom::key_value_pair field : obj) { + if (!first) { + metrics.estimated_inline_len += 2; // ", " + } + first = false; + + // Key length: quotes + key + colon + space + metrics.estimated_inline_len += estimate_string_length(field.key) + 2; + + element_metrics child_metrics = analyze_element(field.value, depth + 1); + metrics.estimated_inline_len += child_metrics.estimated_inline_len; + max_child_complexity = (std::max)(max_child_complexity, child_metrics.complexity); + metrics.child_count++; + metrics.children.push_back(std::move(child_metrics)); + } + + metrics.complexity = 1 + max_child_complexity; + + metrics.can_inline = (metrics.complexity <= current_opts_->max_inline_complexity) && + (metrics.estimated_inline_len <= current_opts_->max_inline_length); + + // Objects use inline or expanded (no table/compact for objects) + if (metrics.child_count == 0 || metrics.can_inline) { + metrics.recommended_layout = layout_mode::INLINE; + } else { + metrics.recommended_layout = layout_mode::EXPANDED; + } + + return metrics; +} + +inline size_t structure_analyzer::estimate_string_length(std::string_view s) const { + size_t len = 2; // quotes + for (char c : s) { + if (c == '"' || c == '\\' || static_cast(c) < 32) { + len += 2; // escape sequence (at least) + } else { + len += 1; + } + } + return len; +} + +inline size_t structure_analyzer::estimate_number_length(double d) const { + if (std::isnan(d) || std::isinf(d)) { + return 4; // "null" for invalid numbers + } + // Rough estimate: up to 17 significant digits + sign + decimal point + exponent + char buf[32]; + int len = snprintf(buf, sizeof(buf), "%.17g", d); + return len > 0 ? static_cast(len) : 20; +} + +inline size_t structure_analyzer::estimate_number_length(int64_t i) const { + if (i == 0) return 1; + // Handle INT64_MIN specially to avoid overflow when negating + if (i == INT64_MIN) return 20; // "-9223372036854775808" is 20 characters + size_t len = (i < 0) ? 1 : 0; // negative sign + int64_t abs_val = (i < 0) ? -i : i; + while (abs_val > 0) { + len++; + abs_val /= 10; + } + return len; +} + +inline size_t structure_analyzer::estimate_number_length(uint64_t u) const { + if (u == 0) return 1; + size_t len = 0; + while (u > 0) { + len++; + u /= 10; + } + return len; +} + +inline bool structure_analyzer::check_array_uniformity(const dom::array& arr, + std::vector& common_keys) const { + common_keys.clear(); + + std::set shared_keys; + dom::object first_obj; + bool have_first = false; + size_t object_count = 0; + + for (dom::element elem : arr) { + if (elem.type() != dom::element_type::OBJECT) { + return false; // Not all elements are objects + } + + dom::object obj; + if (elem.get_object().get(obj) != SUCCESS) { + return false; + } + + std::set current_keys; + for (dom::key_value_pair field : obj) { + current_keys.insert(std::string(field.key)); + } + + if (!have_first) { + shared_keys = current_keys; + first_obj = obj; + have_first = true; + } else { + // Check similarity threshold against the first object + double similarity = compute_object_similarity(first_obj, obj); + if (similarity < current_opts_->table_similarity_threshold) { + return false; // Objects are too dissimilar for table format + } + + // Intersect with current keys + std::set intersection; + std::set_intersection(shared_keys.begin(), shared_keys.end(), + current_keys.begin(), current_keys.end(), + std::inserter(intersection, intersection.begin())); + shared_keys = intersection; + } + + object_count++; + } + + if (object_count < current_opts_->min_table_rows) { + return false; + } + + // Require at least one common key for table formatting + if (shared_keys.empty()) { + return false; + } + + common_keys.assign(shared_keys.begin(), shared_keys.end()); + return true; +} + +inline double structure_analyzer::compute_object_similarity(const dom::object& a, + const dom::object& b) const { + std::set keys_a, keys_b; + for (dom::key_value_pair field : a) { + keys_a.insert(std::string(field.key)); + } + for (dom::key_value_pair field : b) { + keys_b.insert(std::string(field.key)); + } + + std::set intersection; + std::set_intersection(keys_a.begin(), keys_a.end(), + keys_b.begin(), keys_b.end(), + std::inserter(intersection, intersection.begin())); + + std::set union_set; + std::set_union(keys_a.begin(), keys_a.end(), + keys_b.begin(), keys_b.end(), + std::inserter(union_set, union_set.begin())); + + if (union_set.empty()) return 1.0; + return static_cast(intersection.size()) / static_cast(union_set.size()); +} + +inline layout_mode structure_analyzer::decide_layout(const element_metrics& metrics, + size_t depth, + size_t available_width) const { + if (metrics.child_count == 0) { + return layout_mode::INLINE; + } + + // Check inline feasibility + size_t indent_width = depth * current_opts_->indent_spaces; + if (metrics.can_inline && + metrics.estimated_inline_len + indent_width <= available_width) { + return layout_mode::INLINE; + } + + // Check table mode + if (metrics.is_uniform_array && !metrics.common_keys.empty()) { + return layout_mode::TABLE; + } + + // Check compact multiline + if (current_opts_->enable_compact_multiline && + metrics.complexity <= current_opts_->max_compact_array_complexity + 1) { + return layout_mode::COMPACT_MULTILINE; + } + + return layout_mode::EXPANDED; +} + +// +// Fractured Formatter Implementation +// + +inline fractured_formatter::fractured_formatter(const fractured_json_options& opts) + : options_(opts), column_widths_{} {} + +simdjson_inline void fractured_formatter::print_newline() { + if (current_layout_ == layout_mode::INLINE) { + return; // No newlines in inline mode + } + one_char('\n'); + current_line_length_ = 0; +} + +simdjson_inline void fractured_formatter::print_indents(size_t depth) { + if (current_layout_ == layout_mode::INLINE) { + return; // No indentation in inline mode + } + for (size_t i = 0; i < depth * options_.indent_spaces; i++) { + one_char(' '); + current_line_length_++; + } +} + +simdjson_inline void fractured_formatter::print_space() { + one_char(' '); + current_line_length_++; +} + +inline void fractured_formatter::set_layout_mode(layout_mode mode) { + current_layout_ = mode; +} + +inline layout_mode fractured_formatter::get_layout_mode() const { + return current_layout_; +} + +inline void fractured_formatter::set_depth(size_t depth) { + current_depth_ = depth; +} + +inline size_t fractured_formatter::get_depth() const { + return current_depth_; +} + +inline void fractured_formatter::track_line_length(size_t chars) { + current_line_length_ += chars; +} + +inline void fractured_formatter::reset_line_length() { + current_line_length_ = 0; +} + +inline size_t fractured_formatter::get_line_length() const { + return current_line_length_; +} + +inline bool fractured_formatter::should_break_line(size_t upcoming_length) const { + return (current_line_length_ + upcoming_length) > options_.max_total_line_length; +} + +inline const fractured_json_options& fractured_formatter::options() const { + return options_; +} + +inline void fractured_formatter::begin_table_row() { + in_table_mode_ = true; + current_column_ = 0; +} + +inline void fractured_formatter::end_table_row() { + in_table_mode_ = false; + current_column_ = 0; +} + +inline void fractured_formatter::set_column_widths(const std::vector& widths) { + column_widths_ = widths; +} + +inline size_t fractured_formatter::get_column_index() const { + return current_column_; +} + +inline void fractured_formatter::next_column() { + current_column_++; +} + +inline void fractured_formatter::align_to_column_width(size_t actual_width) { + if (current_column_ < column_widths_.size()) { + size_t target_width = column_widths_[current_column_]; + while (actual_width < target_width) { + one_char(' '); + actual_width++; + current_line_length_++; + } + } +} + +// +// Fractured String Builder Implementation +// + +inline fractured_string_builder::fractured_string_builder(const fractured_json_options& opts) + : format_(opts), analyzer_{}, options_(opts) {} + +inline void fractured_string_builder::append(const dom::element& value) { + // Phase 1: Analyze structure (metrics tree is built recursively) + element_metrics root_metrics = analyzer_.analyze(value, options_); + + // Phase 2: Format using metrics tree (passed through recursion) + format_element(value, root_metrics, 0); +} + +inline void fractured_string_builder::append(const dom::array& value) { + // Analyze the array to get proper metrics with children + element_metrics metrics = analyzer_.analyze_array(value, options_); + format_array(value, metrics, 0); +} + +inline void fractured_string_builder::append(const dom::object& value) { + // Analyze the object to get proper metrics with children + element_metrics metrics = analyzer_.analyze_object(value, options_); + format_object(value, metrics, 0); +} + +simdjson_inline void fractured_string_builder::clear() { + format_.clear(); + analyzer_.clear(); +} + +simdjson_inline std::string_view fractured_string_builder::str() const { + return format_.str(); +} + +inline void fractured_string_builder::format_element(const dom::element& elem, + const element_metrics& metrics, + size_t depth) { + switch (elem.type()) { + case dom::element_type::ARRAY: { + dom::array arr; + if (elem.get_array().get(arr) == SUCCESS) { + format_array(arr, metrics, depth); + } + break; + } + case dom::element_type::OBJECT: { + dom::object obj; + if (elem.get_object().get(obj) == SUCCESS) { + format_object(obj, metrics, depth); + } + break; + } + default: + format_scalar(elem); + break; + } +} + +inline void fractured_string_builder::format_array(const dom::array& arr, + const element_metrics& metrics, + size_t depth) { + switch (metrics.recommended_layout) { + case layout_mode::INLINE: + format_array_inline(arr, metrics); + break; + case layout_mode::COMPACT_MULTILINE: + format_array_compact_multiline(arr, metrics, depth); + break; + case layout_mode::TABLE: + format_array_as_table(arr, metrics, depth); + break; + case layout_mode::EXPANDED: + default: + format_array_expanded(arr, metrics, depth); + break; + } +} + +inline void fractured_string_builder::format_array_inline(const dom::array& arr, + const element_metrics& metrics) { + layout_mode prev_layout = format_.get_layout_mode(); + format_.set_layout_mode(layout_mode::INLINE); + + format_.start_array(); + + bool first = true; + bool empty = true; + size_t child_idx = 0; + for (dom::element elem : arr) { + empty = false; + if (!first) { + format_.comma(); + if (options_.comma_padding) { + format_.print_space(); + } + } else if (options_.simple_bracket_padding) { + format_.print_space(); + } + first = false; + const element_metrics& child_metrics = (child_idx < metrics.children.size()) + ? metrics.children[child_idx] : element_metrics{}; + format_element(elem, child_metrics, 0); + child_idx++; + } + + if (options_.simple_bracket_padding && !empty) { + format_.print_space(); + } + format_.end_array(); + + format_.set_layout_mode(prev_layout); +} + +inline void fractured_string_builder::format_array_compact_multiline(const dom::array& arr, + const element_metrics& metrics, + size_t depth) { + format_.start_array(); + format_.print_newline(); + format_.print_indents(depth + 1); + + size_t items_on_line = 0; + bool first = true; + size_t child_idx = 0; + + for (dom::element elem : arr) { + if (!first) { + format_.comma(); + + // Check if we should break to new line + if (items_on_line >= options_.max_items_per_line || + format_.should_break_line(20)) { // 20 is rough estimate for next item + format_.print_newline(); + format_.print_indents(depth + 1); + items_on_line = 0; + } else if (options_.comma_padding) { + format_.print_space(); + } + } + first = false; + + // Format element inline + layout_mode prev_layout = format_.get_layout_mode(); + format_.set_layout_mode(layout_mode::INLINE); + const element_metrics& child_metrics = (child_idx < metrics.children.size()) + ? metrics.children[child_idx] : element_metrics{}; + format_element(elem, child_metrics, depth + 1); + format_.set_layout_mode(prev_layout); + + items_on_line++; + child_idx++; + } + + format_.print_newline(); + format_.print_indents(depth); + format_.end_array(); +} + +inline void fractured_string_builder::format_array_as_table(const dom::array& arr, + const element_metrics& metrics, + size_t depth) { + const std::vector& columns = metrics.common_keys; + if (columns.empty()) { + format_array_expanded(arr, metrics, depth); + return; + } + + // Calculate column widths for alignment + std::vector col_widths = calculate_column_widths(arr, columns); + format_.set_column_widths(col_widths); + + format_.start_array(); + format_.print_newline(); + + bool first_row = true; + size_t child_idx = 0; + for (dom::element elem : arr) { + if (!first_row) { + format_.comma(); + format_.print_newline(); + } + first_row = false; + + format_.print_indents(depth + 1); + format_.begin_table_row(); + + // Format object as inline with aligned columns + dom::object obj; + if (elem.get_object().get(obj) != SUCCESS) { + child_idx++; + continue; + } + + // Get child metrics for this row (object) + const element_metrics& row_metrics = (child_idx < metrics.children.size()) + ? metrics.children[child_idx] : element_metrics{}; + + format_.start_object(); + if (options_.simple_bracket_padding) { + format_.print_space(); + } + + bool first_col = true; + const size_t num_columns = columns.size(); + + for (size_t col_idx = 0; col_idx < num_columns; col_idx++) { + const std::string& key = columns[col_idx]; + const bool is_last_col = (col_idx == num_columns - 1); + + if (!first_col) { + format_.comma(); + if (options_.comma_padding) { + format_.print_space(); + } + } + first_col = false; + + // Write key + format_.key(key); + if (options_.colon_padding) { + format_.print_space(); + } + + // Find the value for this key and its metrics + dom::element value; + bool found = false; + size_t field_idx = 0; + for (dom::key_value_pair field : obj) { + if (field.key == key) { + value = field.value; + found = true; + break; + } + field_idx++; + } + + // Write value + if (found) { + layout_mode prev_layout = format_.get_layout_mode(); + format_.set_layout_mode(layout_mode::INLINE); + const element_metrics& value_metrics = (field_idx < row_metrics.children.size()) + ? row_metrics.children[field_idx] : element_metrics{}; + format_element(value, value_metrics, depth + 1); + format_.set_layout_mode(prev_layout); + } else { + format_.null_atom(); + } + + // Only pad non-last columns to align values across rows + if (!is_last_col) { + size_t actual_len = found ? measure_value_length(value) : 4; // 4 for "null" + size_t target_width = col_widths[col_idx]; + while (actual_len < target_width) { + format_.one_char(' '); + actual_len++; + } + } + + format_.next_column(); + } + + if (options_.simple_bracket_padding) { + format_.print_space(); + } + format_.end_object(); + format_.end_table_row(); + child_idx++; + } + + format_.print_newline(); + format_.print_indents(depth); + format_.end_array(); +} + +inline void fractured_string_builder::format_array_expanded(const dom::array& arr, + const element_metrics& metrics, + size_t depth) { + format_.start_array(); + + bool empty = true; + bool first = true; + size_t child_idx = 0; + + for (dom::element elem : arr) { + empty = false; + if (!first) { + format_.comma(); + } + first = false; + + format_.print_newline(); + format_.print_indents(depth + 1); + const element_metrics& child_metrics = (child_idx < metrics.children.size()) + ? metrics.children[child_idx] : element_metrics{}; + format_element(elem, child_metrics, depth + 1); + child_idx++; + } + + if (!empty) { + format_.print_newline(); + format_.print_indents(depth); + } + format_.end_array(); +} + +inline void fractured_string_builder::format_object(const dom::object& obj, + const element_metrics& metrics, + size_t depth) { + if (metrics.recommended_layout == layout_mode::INLINE || metrics.can_inline) { + format_object_inline(obj, metrics); + } else { + format_object_expanded(obj, metrics, depth); + } +} + +inline void fractured_string_builder::format_object_inline(const dom::object& obj, + const element_metrics& metrics) { + layout_mode prev_layout = format_.get_layout_mode(); + format_.set_layout_mode(layout_mode::INLINE); + + format_.start_object(); + + bool empty = true; + bool first = true; + size_t child_idx = 0; + + for (dom::key_value_pair field : obj) { + empty = false; + if (!first) { + format_.comma(); + if (options_.comma_padding) { + format_.print_space(); + } + } else if (options_.simple_bracket_padding) { + format_.print_space(); + } + first = false; + + format_.key(field.key); + if (options_.colon_padding) { + format_.print_space(); + } + const element_metrics& child_metrics = (child_idx < metrics.children.size()) + ? metrics.children[child_idx] : element_metrics{}; + format_element(field.value, child_metrics, 0); + child_idx++; + } + + if (options_.simple_bracket_padding && !empty) { + format_.print_space(); + } + format_.end_object(); + + format_.set_layout_mode(prev_layout); +} + +inline void fractured_string_builder::format_object_expanded(const dom::object& obj, + const element_metrics& metrics, + size_t depth) { + format_.start_object(); + + bool empty = true; + bool first = true; + size_t child_idx = 0; + + for (dom::key_value_pair field : obj) { + empty = false; + if (!first) { + format_.comma(); + } + first = false; + + format_.print_newline(); + format_.print_indents(depth + 1); + format_.key(field.key); + if (options_.colon_padding) { + format_.print_space(); + } + const element_metrics& child_metrics = (child_idx < metrics.children.size()) + ? metrics.children[child_idx] : element_metrics{}; + format_element(field.value, child_metrics, depth + 1); + child_idx++; + } + + if (!empty) { + format_.print_newline(); + format_.print_indents(depth); + } + format_.end_object(); +} + +inline void fractured_string_builder::format_scalar(const dom::element& elem) { + switch (elem.type()) { + case dom::element_type::STRING: { + std::string_view str; + if (elem.get_string().get(str) == SUCCESS) { + format_.string(str); + } + break; + } + case dom::element_type::INT64: { + int64_t val; + if (elem.get_int64().get(val) == SUCCESS) { + format_.number(val); + } + break; + } + case dom::element_type::UINT64: { + uint64_t val; + if (elem.get_uint64().get(val) == SUCCESS) { + format_.number(val); + } + break; + } + case dom::element_type::DOUBLE: { + double val; + if (elem.get_double().get(val) == SUCCESS) { + format_.number(val); + } + break; + } + case dom::element_type::BOOL: { + bool val; + if (elem.get_bool().get(val) == SUCCESS) { + val ? format_.true_atom() : format_.false_atom(); + } + break; + } + case dom::element_type::NULL_VALUE: + format_.null_atom(); + break; + default: + break; + } +} + +inline size_t fractured_string_builder::measure_value_length(const dom::element& elem) const { + switch (elem.type()) { + case dom::element_type::STRING: { + std::string_view str; + if (elem.get_string().get(str) == SUCCESS) { + // Count actual escaped length + size_t len = 2; // quotes + for (char c : str) { + if (c == '"' || c == '\\' || static_cast(c) < 32) { + len += 2; // escape sequence + } else { + len += 1; + } + } + return len; + } + return 2; + } + case dom::element_type::INT64: { + int64_t val; + if (elem.get_int64().get(val) == SUCCESS) { + if (val == 0) return 1; + // Handle INT64_MIN specially to avoid overflow when negating + if (val == INT64_MIN) return 20; // "-9223372036854775808" is 20 characters + size_t len = (val < 0) ? 1 : 0; + int64_t abs_val = (val < 0) ? -val : val; + while (abs_val > 0) { len++; abs_val /= 10; } + return len; + } + return 1; + } + case dom::element_type::UINT64: { + uint64_t val; + if (elem.get_uint64().get(val) == SUCCESS) { + if (val == 0) return 1; + size_t len = 0; + while (val > 0) { len++; val /= 10; } + return len; + } + return 1; + } + case dom::element_type::DOUBLE: { + double val; + if (elem.get_double().get(val) == SUCCESS) { + char buf[32]; + int len = snprintf(buf, sizeof(buf), "%.17g", val); + return len > 0 ? static_cast(len) : 1; + } + return 1; + } + case dom::element_type::BOOL: { + bool val; + if (elem.get_bool().get(val) == SUCCESS) { + return val ? 4 : 5; // "true" or "false" + } + return 5; + } + case dom::element_type::NULL_VALUE: + return 4; // "null" + default: + return 4; + } +} + +inline std::vector fractured_string_builder::calculate_column_widths( + const dom::array& arr, + const std::vector& columns) const { + + std::vector widths(columns.size(), 0); + + for (dom::element elem : arr) { + dom::object obj; + if (elem.get_object().get(obj) != SUCCESS) { + continue; + } + + for (size_t col_idx = 0; col_idx < columns.size(); col_idx++) { + const std::string& key = columns[col_idx]; + + for (dom::key_value_pair field : obj) { + if (field.key == key) { + // Measure actual value length + size_t len = measure_value_length(field.value); + widths[col_idx] = (std::max)(widths[col_idx], len); + break; + } + } + } + } + + return widths; +} + +} // namespace internal + +// +// Public API Implementation +// + +template +std::string fractured_json(T x) { + return fractured_json(x, fractured_json_options{}); +} + +template +std::string fractured_json(T x, const fractured_json_options& options) { + internal::fractured_string_builder sb(options); + sb.append(x); + std::string_view result = sb.str(); + return std::string(result.data(), result.size()); +} + +#if SIMDJSON_EXCEPTIONS +template +std::string fractured_json(simdjson_result x) { + if (x.error()) { + throw simdjson_error(x.error()); + } + return fractured_json(x.value()); +} + +template +std::string fractured_json(simdjson_result x, const fractured_json_options& options) { + if (x.error()) { + throw simdjson_error(x.error()); + } + return fractured_json(x.value(), options); +} +#endif + +// Explicit template instantiations for common types +template std::string fractured_json(dom::element x); +template std::string fractured_json(dom::element x, const fractured_json_options& options); +template std::string fractured_json(dom::array x); +template std::string fractured_json(dom::array x, const fractured_json_options& options); +template std::string fractured_json(dom::object x); +template std::string fractured_json(dom::object x, const fractured_json_options& options); + +#if SIMDJSON_EXCEPTIONS +template std::string fractured_json(simdjson_result x); +template std::string fractured_json(simdjson_result x, const fractured_json_options& options); +#endif + +// +// String-based API for formatting any JSON string +// + +inline std::string fractured_json_string(std::string_view json_str) { + return fractured_json_string(json_str, fractured_json_options{}); +} + +inline std::string fractured_json_string(std::string_view json_str, + const fractured_json_options& options) { + // Parse the JSON string + dom::parser parser; + dom::element doc; + // Need to pad the string for simdjson + auto padded = padded_string(json_str); + auto error = parser.parse(padded).get(doc); + if (error) { + // If parsing fails, return the original string + return std::string(json_str); + } + return fractured_json(doc, options); +} + +} // namespace simdjson + +#endif // SIMDJSON_DOM_FRACTURED_JSON_INL_H +/* end file simdjson/dom/fractured_json-inl.h */ + +#endif // SIMDJSON_DOM_H +/* end file simdjson/dom.h */ +#if SIMDJSON_FEATURE_BUILDER_API +/* including simdjson/builder.h: #include "simdjson/builder.h" */ +/* begin file simdjson/builder.h */ +#ifndef SIMDJSON_BUILDER_H +#define SIMDJSON_BUILDER_H + +/* including simdjson/builtin/builder.h: #include "simdjson/builtin/builder.h" */ +/* begin file simdjson/builtin/builder.h */ +#ifndef SIMDJSON_BUILTIN_BUILDER_H +#define SIMDJSON_BUILTIN_BUILDER_H + +/* skipped duplicate #include "simdjson/builtin.h" */ /* skipped duplicate #include "simdjson/builtin/base.h" */ /* including simdjson/generic/builder/dependencies.h: #include "simdjson/generic/builder/dependencies.h" */ @@ -61182,6 +61355,10 @@ namespace simdjson { #endif // SIMDJSON_BUILDER_H /* end file simdjson/builder.h */ +#endif // SIMDJSON_FEATURE_BUILDER_API +#endif // SIMDJSON_FEATURE_DOM_API + +#if SIMDJSON_FEATURE_ONDEMAND_API /* including simdjson/ondemand.h: #include "simdjson/ondemand.h" */ /* begin file simdjson/ondemand.h */ #ifndef SIMDJSON_ONDEMAND_H @@ -61207,6 +61384,7 @@ namespace simdjson { // Internal headers needed for ondemand generics. // All includes not under simdjson/generic/ondemand must be here! // Otherwise, amalgamation will fail. +/* skipped duplicate #include "simdjson/feature_macros.h" */ /* skipped duplicate #include "simdjson/dom/base.h" // for MINIMAL_DOCUMENT_CAPACITY */ /* skipped duplicate #include "simdjson/implementation.h" */ /* skipped duplicate #include "simdjson/padded_string.h" */ @@ -62102,7 +62280,13 @@ simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *ds /* end file simdjson/arm64/begin.h */ /* including simdjson/generic/ondemand/amalgamated.h for arm64: #include "simdjson/generic/ondemand/amalgamated.h" */ /* begin file simdjson/generic/ondemand/amalgamated.h for arm64 */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_BUILDER_DEPENDENCIES_H) +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_FEATURE_ONDEMAND_API + +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_ONDEMAND_DEPENDENCIES_H) #error simdjson/generic/ondemand/dependencies.h must be included before simdjson/generic/ondemand/amalgamated.h! #endif @@ -75119,6 +75303,8 @@ inline simdjson_result<::simdjson::arm64::ondemand::value> at_pointer_compiled(D /* end file simdjson/generic/ondemand/compile_time_accessors.h for arm64 */ +#endif // SIMDJSON_FEATURE_ONDEMAND_API + /* end file simdjson/generic/ondemand/amalgamated.h for arm64 */ /* including simdjson/arm64/end.h: #include "simdjson/arm64/end.h" */ /* begin file simdjson/arm64/end.h */ @@ -75378,7 +75564,13 @@ simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t /* end file simdjson/fallback/begin.h */ /* including simdjson/generic/ondemand/amalgamated.h for fallback: #include "simdjson/generic/ondemand/amalgamated.h" */ /* begin file simdjson/generic/ondemand/amalgamated.h for fallback */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_BUILDER_DEPENDENCIES_H) +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_FEATURE_ONDEMAND_API + +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_ONDEMAND_DEPENDENCIES_H) #error simdjson/generic/ondemand/dependencies.h must be included before simdjson/generic/ondemand/amalgamated.h! #endif @@ -88395,6 +88587,8 @@ inline simdjson_result<::simdjson::fallback::ondemand::value> at_pointer_compile /* end file simdjson/generic/ondemand/compile_time_accessors.h for fallback */ +#endif // SIMDJSON_FEATURE_ONDEMAND_API + /* end file simdjson/generic/ondemand/amalgamated.h for fallback */ /* including simdjson/fallback/end.h: #include "simdjson/fallback/end.h" */ /* begin file simdjson/fallback/end.h */ @@ -89141,7 +89335,13 @@ simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *ds /* end file simdjson/haswell/begin.h */ /* including simdjson/generic/ondemand/amalgamated.h for haswell: #include "simdjson/generic/ondemand/amalgamated.h" */ /* begin file simdjson/generic/ondemand/amalgamated.h for haswell */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_BUILDER_DEPENDENCIES_H) +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_FEATURE_ONDEMAND_API + +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_ONDEMAND_DEPENDENCIES_H) #error simdjson/generic/ondemand/dependencies.h must be included before simdjson/generic/ondemand/amalgamated.h! #endif @@ -102158,6 +102358,8 @@ inline simdjson_result<::simdjson::haswell::ondemand::value> at_pointer_compiled /* end file simdjson/generic/ondemand/compile_time_accessors.h for haswell */ +#endif // SIMDJSON_FEATURE_ONDEMAND_API + /* end file simdjson/generic/ondemand/amalgamated.h for haswell */ /* including simdjson/haswell/end.h: #include "simdjson/haswell/end.h" */ /* begin file simdjson/haswell/end.h */ @@ -102904,7 +103106,13 @@ simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t /* end file simdjson/icelake/begin.h */ /* including simdjson/generic/ondemand/amalgamated.h for icelake: #include "simdjson/generic/ondemand/amalgamated.h" */ /* begin file simdjson/generic/ondemand/amalgamated.h for icelake */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_BUILDER_DEPENDENCIES_H) +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_FEATURE_ONDEMAND_API + +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_ONDEMAND_DEPENDENCIES_H) #error simdjson/generic/ondemand/dependencies.h must be included before simdjson/generic/ondemand/amalgamated.h! #endif @@ -115921,6 +116129,8 @@ inline simdjson_result<::simdjson::icelake::ondemand::value> at_pointer_compiled /* end file simdjson/generic/ondemand/compile_time_accessors.h for icelake */ +#endif // SIMDJSON_FEATURE_ONDEMAND_API + /* end file simdjson/generic/ondemand/amalgamated.h for icelake */ /* including simdjson/icelake/end.h: #include "simdjson/icelake/end.h" */ /* begin file simdjson/icelake/end.h */ @@ -116782,7 +116992,13 @@ simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *ds /* end file simdjson/ppc64/begin.h */ /* including simdjson/generic/ondemand/amalgamated.h for ppc64: #include "simdjson/generic/ondemand/amalgamated.h" */ /* begin file simdjson/generic/ondemand/amalgamated.h for ppc64 */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_BUILDER_DEPENDENCIES_H) +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_FEATURE_ONDEMAND_API + +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_ONDEMAND_DEPENDENCIES_H) #error simdjson/generic/ondemand/dependencies.h must be included before simdjson/generic/ondemand/amalgamated.h! #endif @@ -129799,6 +130015,8 @@ inline simdjson_result<::simdjson::ppc64::ondemand::value> at_pointer_compiled(D /* end file simdjson/generic/ondemand/compile_time_accessors.h for ppc64 */ +#endif // SIMDJSON_FEATURE_ONDEMAND_API + /* end file simdjson/generic/ondemand/amalgamated.h for ppc64 */ /* including simdjson/ppc64/end.h: #include "simdjson/ppc64/end.h" */ /* begin file simdjson/ppc64/end.h */ @@ -130977,7 +131195,13 @@ simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *ds /* end file simdjson/westmere/begin.h */ /* including simdjson/generic/ondemand/amalgamated.h for westmere: #include "simdjson/generic/ondemand/amalgamated.h" */ /* begin file simdjson/generic/ondemand/amalgamated.h for westmere */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_BUILDER_DEPENDENCIES_H) +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_FEATURE_ONDEMAND_API + +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_ONDEMAND_DEPENDENCIES_H) #error simdjson/generic/ondemand/dependencies.h must be included before simdjson/generic/ondemand/amalgamated.h! #endif @@ -143994,6 +144218,8 @@ inline simdjson_result<::simdjson::westmere::ondemand::value> at_pointer_compile /* end file simdjson/generic/ondemand/compile_time_accessors.h for westmere */ +#endif // SIMDJSON_FEATURE_ONDEMAND_API + /* end file simdjson/generic/ondemand/amalgamated.h for westmere */ /* including simdjson/westmere/end.h: #include "simdjson/westmere/end.h" */ /* begin file simdjson/westmere/end.h */ @@ -144646,7 +144872,13 @@ simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *ds /* end file simdjson/lsx/begin.h */ /* including simdjson/generic/ondemand/amalgamated.h for lsx: #include "simdjson/generic/ondemand/amalgamated.h" */ /* begin file simdjson/generic/ondemand/amalgamated.h for lsx */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_BUILDER_DEPENDENCIES_H) +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_FEATURE_ONDEMAND_API + +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_ONDEMAND_DEPENDENCIES_H) #error simdjson/generic/ondemand/dependencies.h must be included before simdjson/generic/ondemand/amalgamated.h! #endif @@ -157663,6 +157895,8 @@ inline simdjson_result<::simdjson::lsx::ondemand::value> at_pointer_compiled(Doc /* end file simdjson/generic/ondemand/compile_time_accessors.h for lsx */ +#endif // SIMDJSON_FEATURE_ONDEMAND_API + /* end file simdjson/generic/ondemand/amalgamated.h for lsx */ /* including simdjson/lsx/end.h: #include "simdjson/lsx/end.h" */ /* begin file simdjson/lsx/end.h */ @@ -158338,7 +158572,13 @@ simdjson_inline escaping escaping::copy_and_find(const uint8_t *src, uint8_t *ds /* end file simdjson/lasx/begin.h */ /* including simdjson/generic/ondemand/amalgamated.h for lasx: #include "simdjson/generic/ondemand/amalgamated.h" */ /* begin file simdjson/generic/ondemand/amalgamated.h for lasx */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_BUILDER_DEPENDENCIES_H) +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_FEATURE_ONDEMAND_API + +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_ONDEMAND_DEPENDENCIES_H) #error simdjson/generic/ondemand/dependencies.h must be included before simdjson/generic/ondemand/amalgamated.h! #endif @@ -171355,6 +171595,8 @@ inline simdjson_result<::simdjson::lasx::ondemand::value> at_pointer_compiled(Do /* end file simdjson/generic/ondemand/compile_time_accessors.h for lasx */ +#endif // SIMDJSON_FEATURE_ONDEMAND_API + /* end file simdjson/generic/ondemand/amalgamated.h for lasx */ /* including simdjson/lasx/end.h: #include "simdjson/lasx/end.h" */ /* begin file simdjson/lasx/end.h */ @@ -172034,7 +172276,13 @@ simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t /* end file simdjson/rvv-vls/begin.h */ /* including simdjson/generic/ondemand/amalgamated.h for rvv_vls: #include "simdjson/generic/ondemand/amalgamated.h" */ /* begin file simdjson/generic/ondemand/amalgamated.h for rvv_vls */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_BUILDER_DEPENDENCIES_H) +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/feature_macros.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_FEATURE_ONDEMAND_API + +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_ONDEMAND_DEPENDENCIES_H) #error simdjson/generic/ondemand/dependencies.h must be included before simdjson/generic/ondemand/amalgamated.h! #endif @@ -185051,6 +185299,8 @@ inline simdjson_result<::simdjson::rvv_vls::ondemand::value> at_pointer_compiled /* end file simdjson/generic/ondemand/compile_time_accessors.h for rvv_vls */ +#endif // SIMDJSON_FEATURE_ONDEMAND_API + /* end file simdjson/generic/ondemand/amalgamated.h for rvv_vls */ /* including simdjson/rvv-vls/end.h: #include "simdjson/rvv-vls/end.h" */ /* begin file simdjson/rvv-vls/end.h */ @@ -185086,12 +185336,14 @@ namespace simdjson { * @copydoc simdjson::builtin::ondemand */ namespace ondemand = builtin::ondemand; +#if SIMDJSON_FEATURE_BUILDER_API /** * @copydoc simdjson::builtin::builder */ namespace builder = builtin::builder; +#endif // SIMDJSON_FEATURE_BUILDER_API -#if SIMDJSON_STATIC_REFLECTION +#if SIMDJSON_STATIC_REFLECTION && SIMDJSON_FEATURE_BUILDER_API /** * Create a JSON string from any user-defined type using static reflection. * Only available when SIMDJSON_STATIC_REFLECTION is enabled. @@ -185116,6 +185368,9 @@ namespace simdjson { #endif // SIMDJSON_ONDEMAND_H /* end file simdjson/ondemand.h */ +#endif // SIMDJSON_FEATURE_ONDEMAND_API + +#if SIMDJSON_FEATURE_DOM_API && SIMDJSON_FEATURE_ONDEMAND_API /* including simdjson/convert.h: #include "simdjson/convert.h" */ /* begin file simdjson/convert.h */ @@ -185378,8 +185633,8 @@ inline auto to_adaptor::operator()(ondemand::parser &parser, std::string str) #endif // SIMDJSON_SUPPORTS_CONCEPTS #endif // SIMDJSON_CONVERT_INL_H /* end file simdjson/convert-inl.h */ +#endif // SIMDJSON_FEATURE_DOM_API && SIMDJSON_FEATURE_ONDEMAND_API -// Compile-time JSON parsing (C++26 P2996 reflection) /* including simdjson/compile_time_json.h: #include "simdjson/compile_time_json.h" */ /* begin file simdjson/compile_time_json.h */ /** @@ -185447,14 +185702,17 @@ namespace compile_time { template consteval auto parse_json(); } // namespace compile_time -} // namespace simdjson +inline namespace literals { template consteval auto operator ""_json() { return simdjson::compile_time::parse_json(); } +} // namespace literals +} // namespace simdjson + #endif // SIMDJSON_STATIC_REFLECTION #endif // SIMDJSON_GENERIC_COMPILE_TIME_JSON_H /* end file simdjson/compile_time_json.h */ diff --git a/singleheader/singleheader.zip b/singleheader/singleheader.zip index 8ce2169b4..9687225c3 100644 Binary files a/singleheader/singleheader.zip and b/singleheader/singleheader.zip differ diff --git a/src/arm64.cpp b/src/arm64.cpp index e017ce016..e996c6406 100644 --- a/src/arm64.cpp +++ b/src/arm64.cpp @@ -2,6 +2,7 @@ #define SIMDJSON_SRC_ARM64_CPP #ifndef SIMDJSON_CONDITIONAL_INCLUDE +#include "simdjson/feature_macros.h" #include #endif // SIMDJSON_CONDITIONAL_INCLUDE @@ -142,6 +143,7 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t return arm64::stage1::generic_validate_utf8(buf,len); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } @@ -149,6 +151,7 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } +#endif // SIMDJSON_FEATURE_DOM_API SIMDJSON_NO_SANITIZE_MEMORY simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { @@ -159,11 +162,13 @@ simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(con return arm64::stringparsing::parse_wobbly_string(src, dst); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } return stage2(_doc); } +#endif // SIMDJSON_FEATURE_DOM_API } // namespace arm64 } // namespace simdjson diff --git a/src/fallback.cpp b/src/fallback.cpp index 9c3261124..975c6d66e 100644 --- a/src/fallback.cpp +++ b/src/fallback.cpp @@ -1,9 +1,8 @@ #ifndef SIMDJSON_SRC_FALLBACK_CPP #define SIMDJSON_SRC_FALLBACK_CPP -#include "simdjson/feature_macros.h" - #ifndef SIMDJSON_CONDITIONAL_INCLUDE +#include "simdjson/feature_macros.h" #include #endif // SIMDJSON_CONDITIONAL_INCLUDE diff --git a/src/generic/dependencies.h b/src/generic/dependencies.h index ce5a2f011..afa29c686 100644 --- a/src/generic/dependencies.h +++ b/src/generic/dependencies.h @@ -5,6 +5,7 @@ #ifndef SIMDJSON_SRC_GENERIC_DEPENDENCIES_H #define SIMDJSON_SRC_GENERIC_DEPENDENCIES_H +#include "simdjson/feature_macros.h" #include #endif // SIMDJSON_SRC_GENERIC_DEPENDENCIES_H \ No newline at end of file diff --git a/src/haswell.cpp b/src/haswell.cpp index 315b24194..6ae25aed1 100644 --- a/src/haswell.cpp +++ b/src/haswell.cpp @@ -1,9 +1,8 @@ #ifndef SIMDJSON_SRC_HASWELL_CPP #define SIMDJSON_SRC_HASWELL_CPP -#include "simdjson/feature_macros.h" - #ifndef SIMDJSON_CONDITIONAL_INCLUDE +#include "simdjson/feature_macros.h" #include #endif // SIMDJSON_CONDITIONAL_INCLUDE diff --git a/src/icelake.cpp b/src/icelake.cpp index 8e26b7e66..2cd355ceb 100644 --- a/src/icelake.cpp +++ b/src/icelake.cpp @@ -1,9 +1,8 @@ #ifndef SIMDJSON_SRC_ICELAKE_CPP #define SIMDJSON_SRC_ICELAKE_CPP -#include "simdjson/feature_macros.h" - #ifndef SIMDJSON_CONDITIONAL_INCLUDE +#include "simdjson/feature_macros.h" #include #endif // SIMDJSON_CONDITIONAL_INCLUDE diff --git a/src/lasx.cpp b/src/lasx.cpp index 09b945f59..928349653 100644 --- a/src/lasx.cpp +++ b/src/lasx.cpp @@ -2,6 +2,7 @@ #define SIMDJSON_SRC_LASX_CPP #ifndef SIMDJSON_CONDITIONAL_INCLUDE +#include "simdjson/feature_macros.h" #include #endif // SIMDJSON_CONDITIONAL_INCLUDE @@ -102,6 +103,7 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t return lasx::stage1::generic_validate_utf8(buf,len); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } @@ -109,6 +111,7 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } +#endif // SIMDJSON_FEATURE_DOM_API SIMDJSON_NO_SANITIZE_MEMORY simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { @@ -119,11 +122,13 @@ simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(con return lasx::stringparsing::parse_wobbly_string(src, dst); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } return stage2(_doc); } +#endif // SIMDJSON_FEATURE_DOM_API } // namespace lasx } // namespace simdjson diff --git a/src/lsx.cpp b/src/lsx.cpp index e68aada87..215c96249 100644 --- a/src/lsx.cpp +++ b/src/lsx.cpp @@ -2,6 +2,7 @@ #define SIMDJSON_SRC_LSX_CPP #ifndef SIMDJSON_CONDITIONAL_INCLUDE +#include "simdjson/feature_macros.h" #include #endif // SIMDJSON_CONDITIONAL_INCLUDE @@ -106,6 +107,7 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t return lsx::stage1::generic_validate_utf8(buf,len); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } @@ -113,6 +115,7 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } +#endif // SIMDJSON_FEATURE_DOM_API SIMDJSON_NO_SANITIZE_MEMORY simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { @@ -123,11 +126,13 @@ simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(con return lsx::stringparsing::parse_wobbly_string(src, dst); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } return stage2(_doc); } +#endif // SIMDJSON_FEATURE_DOM_API } // namespace lsx } // namespace simdjson diff --git a/src/ppc64.cpp b/src/ppc64.cpp index f7dee707e..8e7aa29b8 100644 --- a/src/ppc64.cpp +++ b/src/ppc64.cpp @@ -2,6 +2,7 @@ #define SIMDJSON_SRC_PPC64_CPP #ifndef SIMDJSON_CONDITIONAL_INCLUDE +#include "simdjson/feature_macros.h" #include #endif // SIMDJSON_CONDITIONAL_INCLUDE @@ -112,6 +113,7 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t return ppc64::stage1::generic_validate_utf8(buf,len); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } @@ -119,6 +121,7 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } +#endif // SIMDJSON_FEATURE_DOM_API SIMDJSON_NO_SANITIZE_MEMORY simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { @@ -129,11 +132,13 @@ simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(con return ppc64::stringparsing::parse_wobbly_string(src, dst); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } return stage2(_doc); } +#endif // SIMDJSON_FEATURE_DOM_API } // namespace ppc64 } // namespace simdjson diff --git a/src/rvv-vls.cpp b/src/rvv-vls.cpp index f603be5b4..83e7e79a5 100644 --- a/src/rvv-vls.cpp +++ b/src/rvv-vls.cpp @@ -2,6 +2,7 @@ #define SIMDJSON_SRC_RVV_VLS_CPP #ifndef SIMDJSON_CONDITIONAL_INCLUDE +#include "simdjson/feature_macros.h" #include #endif // SIMDJSON_CONDITIONAL_INCLUDE @@ -101,6 +102,7 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t return rvv_vls::stage1::generic_validate_utf8(buf,len); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } @@ -108,6 +110,7 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { return stage2::tape_builder::parse_document(*this, _doc); } +#endif // SIMDJSON_FEATURE_DOM_API SIMDJSON_NO_SANITIZE_MEMORY simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { @@ -118,11 +121,13 @@ simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(con return rvv_vls::stringparsing::parse_wobbly_string(src, dst); } +#if SIMDJSON_FEATURE_DOM_API simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } return stage2(_doc); } +#endif // SIMDJSON_FEATURE_DOM_API } // namespace rvv_vls } // namespace simdjson diff --git a/src/westmere.cpp b/src/westmere.cpp index 5357e1071..fe0b4b4fa 100644 --- a/src/westmere.cpp +++ b/src/westmere.cpp @@ -1,9 +1,8 @@ #ifndef SIMDJSON_SRC_WESTMERE_CPP #define SIMDJSON_SRC_WESTMERE_CPP -#include "simdjson/feature_macros.h" - #ifndef SIMDJSON_CONDITIONAL_INCLUDE +#include "simdjson/feature_macros.h" #include #endif // SIMDJSON_CONDITIONAL_INCLUDE diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 855174576..845dca168 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -29,4 +29,5 @@ add_cpp_test(checkimplementation LABELS other per_implementation) add_subdirectory(compilation_failure_tests) add_subdirectory(builder) -add_subdirectory(compile_time) \ No newline at end of file +add_subdirectory(compile_time) +add_subdirectory(feature_flag_tests) \ No newline at end of file diff --git a/tests/builder/CMakeLists.txt b/tests/builder/CMakeLists.txt index df8501cb9..6108d9385 100644 --- a/tests/builder/CMakeLists.txt +++ b/tests/builder/CMakeLists.txt @@ -1,5 +1,8 @@ # All remaining tests link with simdjson proper include_directories(..) +if(NOT SIMDJSON_FEATURE_BUILDER_API) + return() +endif() add_cpp_test(builder_string_builder_tests LABELS ondemand acceptance per_implementation) if(SIMDJSON_STATIC_REFLECTION) add_cpp_test(static_reflection_custom_tests LABELS ondemand acceptance per_implementation) diff --git a/tests/feature_flag_tests/CMakeLists.txt b/tests/feature_flag_tests/CMakeLists.txt new file mode 100644 index 000000000..f4f09d0f4 --- /dev/null +++ b/tests/feature_flag_tests/CMakeLists.txt @@ -0,0 +1,36 @@ +# Feature flag tests: compile and run against the amalgamated single-header file. +# Each test uses target_compile_definitions to set its own feature macros, +# independent of the project-wide settings. + +set(SINGLEHEADER_DIR ${PROJECT_SOURCE_DIR}/singleheader) + +function(add_feature_flag_test TEST_NAME) + cmake_parse_arguments(PARSE_ARGV 1 ARG "" "" "DEFS") + add_executable(${TEST_NAME} ${TEST_NAME}.cpp ${SINGLEHEADER_DIR}/simdjson.cpp) + target_include_directories(${TEST_NAME} PRIVATE ${SINGLEHEADER_DIR}) + target_compile_features(${TEST_NAME} PRIVATE cxx_std_17) + # Override any inherited feature flag definitions with our own. + target_compile_definitions(${TEST_NAME} PRIVATE ${ARG_DEFS}) + # Remove inherited link libraries (simdjson-internal-flags brings -Werror + # which conflicts with the amalgamated header's warning profile). + set_target_properties(${TEST_NAME} PROPERTIES LINK_LIBRARIES "") + add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) + set_property(TEST ${TEST_NAME} APPEND PROPERTY LABELS feature_flags acceptance) +endfunction() + +add_feature_flag_test(dom_only DEFS + SIMDJSON_FEATURE_DOM_API=1 SIMDJSON_FEATURE_ONDEMAND_API=0 SIMDJSON_FEATURE_BUILDER_API=0) +# Note: OnDemand tests also enable DOM because the builtin infrastructure +# is currently included via the DOM/builtin path in the amalgamated header. +add_feature_flag_test(ondemand_only DEFS + SIMDJSON_FEATURE_DOM_API=1 SIMDJSON_FEATURE_ONDEMAND_API=1 SIMDJSON_FEATURE_BUILDER_API=0) +add_feature_flag_test(dom_and_ondemand DEFS + SIMDJSON_FEATURE_DOM_API=1 SIMDJSON_FEATURE_ONDEMAND_API=1 SIMDJSON_FEATURE_BUILDER_API=0) +add_feature_flag_test(builder_and_dom DEFS + SIMDJSON_FEATURE_DOM_API=1 SIMDJSON_FEATURE_ONDEMAND_API=0 SIMDJSON_FEATURE_BUILDER_API=1) +add_feature_flag_test(builder_and_ondemand DEFS + SIMDJSON_FEATURE_DOM_API=1 SIMDJSON_FEATURE_ONDEMAND_API=1 SIMDJSON_FEATURE_BUILDER_API=1) +add_feature_flag_test(builder_dom_and_ondemand DEFS + SIMDJSON_FEATURE_DOM_API=1 SIMDJSON_FEATURE_ONDEMAND_API=1 SIMDJSON_FEATURE_BUILDER_API=1) +add_feature_flag_test(builder_only DEFS + SIMDJSON_FEATURE_DOM_API=1 SIMDJSON_FEATURE_ONDEMAND_API=0 SIMDJSON_FEATURE_BUILDER_API=1) diff --git a/tests/feature_flag_tests/builder_and_dom.cpp b/tests/feature_flag_tests/builder_and_dom.cpp new file mode 100644 index 000000000..f69d9aff1 --- /dev/null +++ b/tests/feature_flag_tests/builder_and_dom.cpp @@ -0,0 +1,42 @@ +// Test: Builder + DOM APIs (OnDemand disabled) +// Feature flags set via target_compile_definitions in CMakeLists.txt +#include "simdjson.h" +#include + +int main() { + auto json = R"({"key": "value"})"_padded; + + // Use DOM API + simdjson::dom::parser parser; + simdjson::dom::element doc; + auto error = parser.parse(json).get(doc); + if (error) { + std::cerr << "DOM parse error: " << error << std::endl; + return EXIT_FAILURE; + } + std::string_view val; + error = doc["key"].get_string().get(val); + if (error || val != "value") { + std::cerr << "DOM key lookup failed" << std::endl; + return EXIT_FAILURE; + } + + // Use Builder API + simdjson::builder::string_builder sb; + sb.start_object(); + sb.append_key_value("greeting", "hello"); + sb.end_object(); + std::string_view result; + error = sb.view().get(result); + if (error) { + std::cerr << "Builder error: " << error << std::endl; + return EXIT_FAILURE; + } + if (result != R"({"greeting":"hello"})") { + std::cerr << "Builder output mismatch: " << result << std::endl; + return EXIT_FAILURE; + } + + std::cout << "Builder+DOM test passed." << std::endl; + return EXIT_SUCCESS; +} diff --git a/tests/feature_flag_tests/builder_and_ondemand.cpp b/tests/feature_flag_tests/builder_and_ondemand.cpp new file mode 100644 index 000000000..f99b2e636 --- /dev/null +++ b/tests/feature_flag_tests/builder_and_ondemand.cpp @@ -0,0 +1,45 @@ +// Test: Builder + OnDemand APIs (DOM also enabled since Builder depends on it) +// Feature flags set via target_compile_definitions in CMakeLists.txt +#include "simdjson.h" +#include + +int main() { + // Use OnDemand API + auto json = R"({"key": "value"})"_padded; + simdjson::ondemand::parser parser; + simdjson::ondemand::document doc; + auto error = parser.iterate(json).get(doc); + if (error) { + std::cerr << "OnDemand parse error: " << error << std::endl; + return EXIT_FAILURE; + } + std::string_view val; + error = doc["key"].get_string().get(val); + if (error || val != "value") { + std::cerr << "OnDemand key lookup failed" << std::endl; + return EXIT_FAILURE; + } + + // Use Builder API + simdjson::builder::string_builder sb; + sb.start_array(); + sb.append(1); + sb.append_comma(); + sb.append(2); + sb.append_comma(); + sb.append(3); + sb.end_array(); + std::string_view result; + error = sb.view().get(result); + if (error) { + std::cerr << "Builder error: " << error << std::endl; + return EXIT_FAILURE; + } + if (result != "[1,2,3]") { + std::cerr << "Builder output mismatch: " << result << std::endl; + return EXIT_FAILURE; + } + + std::cout << "Builder+OnDemand test passed." << std::endl; + return EXIT_SUCCESS; +} diff --git a/tests/feature_flag_tests/builder_dom_and_ondemand.cpp b/tests/feature_flag_tests/builder_dom_and_ondemand.cpp new file mode 100644 index 000000000..cc582c1c6 --- /dev/null +++ b/tests/feature_flag_tests/builder_dom_and_ondemand.cpp @@ -0,0 +1,59 @@ +// Test: All three APIs enabled (Builder + DOM + OnDemand) +// Feature flags set via target_compile_definitions in CMakeLists.txt +#include "simdjson.h" +#include + +int main() { + auto json = R"({"name": "simdjson", "version": 5})"_padded; + + // Use DOM API + simdjson::dom::parser dom_parser; + simdjson::dom::element dom_doc; + auto error = dom_parser.parse(json).get(dom_doc); + if (error) { + std::cerr << "DOM parse error: " << error << std::endl; + return EXIT_FAILURE; + } + std::string_view name; + error = dom_doc["name"].get_string().get(name); + if (error || name != "simdjson") { + std::cerr << "DOM name lookup failed" << std::endl; + return EXIT_FAILURE; + } + + // Use OnDemand API + simdjson::ondemand::parser od_parser; + simdjson::ondemand::document od_doc; + error = od_parser.iterate(json).get(od_doc); + if (error) { + std::cerr << "OnDemand parse error: " << error << std::endl; + return EXIT_FAILURE; + } + std::string_view od_name; + error = od_doc["name"].get_string().get(od_name); + if (error || od_name != "simdjson") { + std::cerr << "OnDemand name lookup failed" << std::endl; + return EXIT_FAILURE; + } + + // Use Builder API + simdjson::builder::string_builder sb; + sb.start_object(); + sb.append_key_value("name", "simdjson"); + sb.append_comma(); + sb.append_key_value("version", 5); + sb.end_object(); + std::string_view result; + error = sb.view().get(result); + if (error) { + std::cerr << "Builder error: " << error << std::endl; + return EXIT_FAILURE; + } + if (result != R"({"name":"simdjson","version":5})") { + std::cerr << "Builder output mismatch: " << result << std::endl; + return EXIT_FAILURE; + } + + std::cout << "Builder+DOM+OnDemand test passed." << std::endl; + return EXIT_SUCCESS; +} diff --git a/tests/feature_flag_tests/builder_only.cpp b/tests/feature_flag_tests/builder_only.cpp new file mode 100644 index 000000000..f9f68fbc1 --- /dev/null +++ b/tests/feature_flag_tests/builder_only.cpp @@ -0,0 +1,27 @@ +// Test: Builder API only (OnDemand disabled, DOM enabled since Builder depends on it) +// Feature flags set via target_compile_definitions in CMakeLists.txt +#include "simdjson.h" +#include + +int main() { + // Use Builder API + simdjson::builder::string_builder sb; + sb.start_object(); + sb.append_key_value("count", 2); + sb.append_comma(); + sb.append_key_value("name", "test"); + sb.end_object(); + std::string_view result; + auto error = sb.view().get(result); + if (error) { + std::cerr << "Builder error: " << error << std::endl; + return EXIT_FAILURE; + } + if (result != R"({"count":2,"name":"test"})") { + std::cerr << "Builder output mismatch: " << result << std::endl; + return EXIT_FAILURE; + } + + std::cout << "Builder-only test passed." << std::endl; + return EXIT_SUCCESS; +} diff --git a/tests/feature_flag_tests/dom_and_ondemand.cpp b/tests/feature_flag_tests/dom_and_ondemand.cpp new file mode 100644 index 000000000..54a617f99 --- /dev/null +++ b/tests/feature_flag_tests/dom_and_ondemand.cpp @@ -0,0 +1,41 @@ +// Test: Both DOM and OnDemand APIs (Builder disabled) +// Feature flags set via target_compile_definitions in CMakeLists.txt +#include "simdjson.h" +#include + +int main() { + auto json = R"({"key": "value", "number": 42})"_padded; + + // Use DOM API + simdjson::dom::parser dom_parser; + simdjson::dom::element dom_doc; + auto error = dom_parser.parse(json).get(dom_doc); + if (error) { + std::cerr << "DOM parse error: " << error << std::endl; + return EXIT_FAILURE; + } + std::string_view dom_val; + error = dom_doc["key"].get_string().get(dom_val); + if (error || dom_val != "value") { + std::cerr << "DOM key lookup failed" << std::endl; + return EXIT_FAILURE; + } + + // Use OnDemand API + simdjson::ondemand::parser od_parser; + simdjson::ondemand::document od_doc; + error = od_parser.iterate(json).get(od_doc); + if (error) { + std::cerr << "OnDemand parse error: " << error << std::endl; + return EXIT_FAILURE; + } + std::string_view od_val; + error = od_doc["key"].get_string().get(od_val); + if (error || od_val != "value") { + std::cerr << "OnDemand key lookup failed" << std::endl; + return EXIT_FAILURE; + } + + std::cout << "DOM+OnDemand test passed." << std::endl; + return EXIT_SUCCESS; +} diff --git a/tests/feature_flag_tests/dom_only.cpp b/tests/feature_flag_tests/dom_only.cpp new file mode 100644 index 000000000..73e47c5be --- /dev/null +++ b/tests/feature_flag_tests/dom_only.cpp @@ -0,0 +1,29 @@ +// Test: DOM API only (OnDemand and Builder disabled) +// Feature flags set via target_compile_definitions in CMakeLists.txt +#include "simdjson.h" +#include + +int main() { + simdjson::dom::parser parser; + auto json = R"({"key": "value", "number": 42})"_padded; + simdjson::dom::element doc; + auto error = parser.parse(json).get(doc); + if (error) { + std::cerr << "DOM parse error: " << error << std::endl; + return EXIT_FAILURE; + } + std::string_view val; + error = doc["key"].get_string().get(val); + if (error || val != "value") { + std::cerr << "DOM key lookup failed" << std::endl; + return EXIT_FAILURE; + } + int64_t num; + error = doc["number"].get_int64().get(num); + if (error || num != 42) { + std::cerr << "DOM number lookup failed" << std::endl; + return EXIT_FAILURE; + } + std::cout << "DOM-only test passed." << std::endl; + return EXIT_SUCCESS; +} diff --git a/tests/feature_flag_tests/ondemand_only.cpp b/tests/feature_flag_tests/ondemand_only.cpp new file mode 100644 index 000000000..802f9e383 --- /dev/null +++ b/tests/feature_flag_tests/ondemand_only.cpp @@ -0,0 +1,25 @@ +// Test: OnDemand API only (Builder disabled) +// Note: DOM must remain enabled because the builtin infrastructure +// (SIMDJSON_BUILTIN_IMPLEMENTATION_IS) is currently included via the DOM path. +// Feature flags set via target_compile_definitions in CMakeLists.txt +#include "simdjson.h" +#include + +int main() { + simdjson::ondemand::parser parser; + auto json = R"({"key": "value", "number": 42})"_padded; + simdjson::ondemand::document doc; + auto error = parser.iterate(json).get(doc); + if (error) { + std::cerr << "OnDemand parse error: " << error << std::endl; + return EXIT_FAILURE; + } + std::string_view val; + error = doc["key"].get_string().get(val); + if (error || val != "value") { + std::cerr << "OnDemand key lookup failed" << std::endl; + return EXIT_FAILURE; + } + std::cout << "OnDemand-only test passed." << std::endl; + return EXIT_SUCCESS; +}