admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

builder -> visitor, parser -> iter

Browse Source
This commit is contained in:
John Keiser
2020-08-06 09:52:26 -07:00
parent 04d39c0961
commit 970dfc9f67
2 changed files with 112 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/generic/stage2/structural_parser.h
+27 -27
View File
@@ -17,7 +17,7 @@ struct structural_parser : structural_iterator {
uint32_t depth{0};
template<bool STREAMING, typename T>
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code walk_document(T &builder) noexcept;
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code walk_document(T &visitor) noexcept;
// For non-streaming, to pass an explicit 0 as next_structural, which enables optimizations
simdjson_really_inline structural_parser(dom_parser_implementation &_dom_parser, uint32_t start_structural_index)
@@ -25,19 +25,19 @@ struct structural_parser : structural_iterator {
}
template<typename T>
SIMDJSON_WARN_UNUSED simdjson_really_inline bool empty_object(T &builder) {
SIMDJSON_WARN_UNUSED simdjson_really_inline bool empty_object(T &visitor) {
if (peek_next_char() == '}') {
advance_char();
builder.empty_object(*this);
visitor.empty_object(*this);
return true;
}
return false;
}
template<typename T>
SIMDJSON_WARN_UNUSED simdjson_really_inline bool empty_array(T &builder) {
SIMDJSON_WARN_UNUSED simdjson_really_inline bool empty_array(T &visitor) {
if (peek_next_char() == ']') {
advance_char();
builder.empty_array(*this);
visitor.empty_array(*this);
return true;
}
return false;
@@ -85,14 +85,14 @@ struct structural_parser : structural_iterator {
}; // struct structural_parser
template<bool STREAMING, typename T>
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code structural_parser::walk_document(T &builder) noexcept {
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code structural_parser::walk_document(T &visitor) noexcept {
logger::log_start();
//
// Start the document
//
if (at_end()) { return EMPTY; }
builder.start_document(*this);
visitor.start_document(*this);
//
// Read first value
@@ -118,9 +118,9 @@ SIMDJSON_WARN_UNUSED simdjson_really_inline error_code structural_parser::walk_d
}
switch (*value) {
case '{': if (!empty_object(builder)) { goto object_begin; }; break;
case '[': if (!empty_array(builder)) { goto array_begin; }; break;
default: SIMDJSON_TRY( builder.parse_root_primitive(*this, value) );
case '{': if (!empty_object(visitor)) { goto object_begin; }; break;
case '[': if (!empty_array(visitor)) { goto array_begin; }; break;
default: SIMDJSON_TRY( visitor.parse_root_primitive(*this, value) );
}
goto document_end;
}
@@ -131,15 +131,15 @@ SIMDJSON_WARN_UNUSED simdjson_really_inline error_code structural_parser::walk_d
object_begin: {
depth++;
if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; }
builder.start_object(*this);
visitor.start_object(*this);
const uint8_t *key = advance();
if (*key != '"') {
log_error("Object does not start with a key");
return TAPE_ERROR;
}
builder.increment_count(*this);
SIMDJSON_TRY( builder.parse_key(*this, key) );
visitor.increment_count(*this);
SIMDJSON_TRY( visitor.parse_key(*this, key) );
goto object_field;
} // object_begin:
@@ -147,23 +147,23 @@ object_field: {
if (simdjson_unlikely( advance_char() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; }
const uint8_t *value = advance();
switch (*value) {
case '{': if (!empty_object(builder)) { goto object_begin; }; break;
case '[': if (!empty_array(builder)) { goto array_begin; }; break;
default: SIMDJSON_TRY( builder.parse_primitive(*this, value) );
case '{': if (!empty_object(visitor)) { goto object_begin; }; break;
case '[': if (!empty_array(visitor)) { goto array_begin; }; break;
default: SIMDJSON_TRY( visitor.parse_primitive(*this, value) );
}
} // object_field:
object_continue: {
switch (advance_char()) {
case ',': {
builder.increment_count(*this);
visitor.increment_count(*this);
const uint8_t *key = advance();
if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; }
SIMDJSON_TRY( builder.parse_key(*this, key) );
SIMDJSON_TRY( visitor.parse_key(*this, key) );
goto object_field;
}
case '}':
builder.end_object(*this);
visitor.end_object(*this);
goto scope_end;
default:
log_error("No comma between object fields");
@@ -184,27 +184,27 @@ scope_end: {
array_begin: {
depth++;
if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; }
builder.start_array(*this);
visitor.start_array(*this);
builder.increment_count(*this);
visitor.increment_count(*this);
} // array_begin:
array_value: {
const uint8_t *value = advance();
switch (*value) {
case '{': if (!empty_object(builder)) { goto object_begin; }; break;
case '[': if (!empty_array(builder)) { goto array_begin; }; break;
default: SIMDJSON_TRY( builder.parse_primitive(*this, value) );
case '{': if (!empty_object(visitor)) { goto object_begin; }; break;
case '[': if (!empty_array(visitor)) { goto array_begin; }; break;
default: SIMDJSON_TRY( visitor.parse_primitive(*this, value) );
}
} // array_value:
array_continue: {
switch (advance_char()) {
case ',':
builder.increment_count(*this);
visitor.increment_count(*this);
goto array_value;
case ']':
builder.end_array(*this);
visitor.end_array(*this);
goto scope_end;
default:
log_error("Missing comma between array values");
@@ -213,7 +213,7 @@ array_continue: {
} // array_continue:
document_end: {
builder.end_document(*this);
visitor.end_document(*this);
return finish<STREAMING>();
} // document_end:
src/generic/stage2/tape_builder.h
+85 -85
View File
@@ -27,98 +27,98 @@ private:
simdjson_really_inline tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {}
simdjson_really_inline error_code parse_root_primitive(structural_parser &parser, const uint8_t *value) {
simdjson_really_inline error_code parse_root_primitive(structural_parser &iter, const uint8_t *value) {
switch (*value) {
case '"': return parse_string(parser, value);
case 't': return parse_root_true_atom(parser, value);
case 'f': return parse_root_false_atom(parser, value);
case 'n': return parse_root_null_atom(parser, value);
case '"': return parse_string(iter, value);
case 't': return parse_root_true_atom(iter, value);
case 'f': return parse_root_false_atom(iter, value);
case 'n': return parse_root_null_atom(iter, value);
case '-':
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
return parse_root_number(parser, value);
return parse_root_number(iter, value);
default:
parser.log_error("Document starts with a non-value character");
iter.log_error("Document starts with a non-value character");
return TAPE_ERROR;
}
}
simdjson_really_inline error_code parse_primitive(structural_parser &parser, const uint8_t *value) {
simdjson_really_inline error_code parse_primitive(structural_parser &iter, const uint8_t *value) {
switch (*value) {
case '"': return parse_string(parser, value);
case 't': return parse_true_atom(parser, value);
case 'f': return parse_false_atom(parser, value);
case 'n': return parse_null_atom(parser, value);
case '"': return parse_string(iter, value);
case 't': return parse_true_atom(iter, value);
case 'f': return parse_false_atom(iter, value);
case 'n': return parse_null_atom(iter, value);
case '-':
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
return parse_number(parser, value);
return parse_number(iter, value);
default:
parser.log_error("Non-value found when value was expected!");
iter.log_error("Non-value found when value was expected!");
return TAPE_ERROR;
}
}
simdjson_really_inline void empty_object(structural_parser &parser) {
parser.log_value("empty object");
empty_container(parser, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT);
simdjson_really_inline void empty_object(structural_parser &iter) {
iter.log_value("empty object");
empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT);
}
simdjson_really_inline void empty_array(structural_parser &parser) {
parser.log_value("empty array");
empty_container(parser, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY);
simdjson_really_inline void empty_array(structural_parser &iter) {
iter.log_value("empty array");
empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY);
}
simdjson_really_inline void start_document(structural_parser &parser) {
parser.log_start_value("document");
start_container(parser);
parser.dom_parser.is_array[parser.depth] = false;
simdjson_really_inline void start_document(structural_parser &iter) {
iter.log_start_value("document");
start_container(iter);
iter.dom_parser.is_array[iter.depth] = false;
}
simdjson_really_inline void start_object(structural_parser &parser) {
parser.log_start_value("object");
start_container(parser);
parser.dom_parser.is_array[parser.depth] = false;
simdjson_really_inline void start_object(structural_parser &iter) {
iter.log_start_value("object");
start_container(iter);
iter.dom_parser.is_array[iter.depth] = false;
}
simdjson_really_inline void start_array(structural_parser &parser) {
parser.log_start_value("array");
start_container(parser);
parser.dom_parser.is_array[parser.depth] = true;
simdjson_really_inline void start_array(structural_parser &iter) {
iter.log_start_value("array");
start_container(iter);
iter.dom_parser.is_array[iter.depth] = true;
}
simdjson_really_inline void end_object(structural_parser &parser) {
parser.log_end_value("object");
end_container(parser, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT);
simdjson_really_inline void end_object(structural_parser &iter) {
iter.log_end_value("object");
end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT);
}
simdjson_really_inline void end_array(structural_parser &parser) {
parser.log_end_value("array");
end_container(parser, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY);
simdjson_really_inline void end_array(structural_parser &iter) {
iter.log_end_value("array");
end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY);
}
simdjson_really_inline void end_document(structural_parser &parser) {
parser.log_end_value("document");
simdjson_really_inline void end_document(structural_parser &iter) {
iter.log_end_value("document");
constexpr uint32_t start_tape_index = 0;
tape.append(start_tape_index, internal::tape_type::ROOT);
tape_writer::write(parser.dom_parser.doc->tape[start_tape_index], next_tape_index(parser), internal::tape_type::ROOT);
tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT);
}
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_key(structural_parser &parser, const uint8_t *value) {
return parse_string(parser, value, true);
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_key(structural_parser &iter, const uint8_t *value) {
return parse_string(iter, value, true);
}
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_string(structural_parser &parser, const uint8_t *value, bool key = false) {
parser.log_value(key ? "key" : "string");
uint8_t *dst = on_start_string(parser);
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_string(structural_parser &iter, const uint8_t *value, bool key = false) {
iter.log_value(key ? "key" : "string");
uint8_t *dst = on_start_string(iter);
dst = stringparsing::parse_string(value, dst);
if (dst == nullptr) {
parser.log_error("Invalid escape in string");
iter.log_error("Invalid escape in string");
return STRING_ERROR;
}
on_end_string(dst);
return SUCCESS;
}
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_number(structural_parser &parser, const uint8_t *value) {
parser.log_value("number");
if (!numberparsing::parse_number(value, tape)) { parser.log_error("Invalid number"); return NUMBER_ERROR; }
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_number(structural_parser &iter, const uint8_t *value) {
iter.log_value("number");
if (!numberparsing::parse_number(value, tape)) { iter.log_error("Invalid number"); return NUMBER_ERROR; }
return SUCCESS;
}
simdjson_really_inline error_code parse_root_number(structural_parser &parser, const uint8_t *value) {
simdjson_really_inline error_code parse_root_number(structural_parser &iter, const uint8_t *value) {
//
// We need to make a copy to make sure that the string is space terminated.
// This is not about padding the input, which should already padded up
@@ -132,97 +132,97 @@ private:
// practice unless you are in the strange scenario where you have many JSON
// documents made of single atoms.
//
uint8_t *copy = static_cast<uint8_t *>(malloc(parser.remaining_len() + SIMDJSON_PADDING));
uint8_t *copy = static_cast<uint8_t *>(malloc(iter.remaining_len() + SIMDJSON_PADDING));
if (copy == nullptr) {
return MEMALLOC;
}
memcpy(copy, value, parser.remaining_len());
memset(copy + parser.remaining_len(), ' ', SIMDJSON_PADDING);
error_code error = parse_number(parser, copy);
memcpy(copy, value, iter.remaining_len());
memset(copy + iter.remaining_len(), ' ', SIMDJSON_PADDING);
error_code error = parse_number(iter, copy);
free(copy);
return error;
}
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_true_atom(structural_parser &parser, const uint8_t *value) {
parser.log_value("true");
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_true_atom(structural_parser &iter, const uint8_t *value) {
iter.log_value("true");
if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; }
tape.append(0, internal::tape_type::TRUE_VALUE);
return SUCCESS;
}
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_root_true_atom(structural_parser &parser, const uint8_t *value) {
parser.log_value("true");
if (!atomparsing::is_valid_true_atom(value, parser.remaining_len())) { return T_ATOM_ERROR; }
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_root_true_atom(structural_parser &iter, const uint8_t *value) {
iter.log_value("true");
if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; }
tape.append(0, internal::tape_type::TRUE_VALUE);
return SUCCESS;
}
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_false_atom(structural_parser &parser, const uint8_t *value) {
parser.log_value("false");
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_false_atom(structural_parser &iter, const uint8_t *value) {
iter.log_value("false");
if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; }
tape.append(0, internal::tape_type::FALSE_VALUE);
return SUCCESS;
}
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_root_false_atom(structural_parser &parser, const uint8_t *value) {
parser.log_value("false");
if (!atomparsing::is_valid_false_atom(value, parser.remaining_len())) { return F_ATOM_ERROR; }
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_root_false_atom(structural_parser &iter, const uint8_t *value) {
iter.log_value("false");
if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; }
tape.append(0, internal::tape_type::FALSE_VALUE);
return SUCCESS;
}
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_null_atom(structural_parser &parser, const uint8_t *value) {
parser.log_value("null");
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_null_atom(structural_parser &iter, const uint8_t *value) {
iter.log_value("null");
if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; }
tape.append(0, internal::tape_type::NULL_VALUE);
return SUCCESS;
}
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_root_null_atom(structural_parser &parser, const uint8_t *value) {
parser.log_value("null");
if (!atomparsing::is_valid_null_atom(value, parser.remaining_len())) { return N_ATOM_ERROR; }
SIMDJSON_WARN_UNUSED simdjson_really_inline error_code parse_root_null_atom(structural_parser &iter, const uint8_t *value) {
iter.log_value("null");
if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; }
tape.append(0, internal::tape_type::NULL_VALUE);
return SUCCESS;
}
// increment_count increments the count of keys in an object or values in an array.
simdjson_really_inline void increment_count(structural_parser &parser) {
parser.dom_parser.open_containers[parser.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1
simdjson_really_inline void increment_count(structural_parser &iter) {
iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1
}
// private:
simdjson_really_inline uint32_t next_tape_index(structural_parser &parser) {
return uint32_t(tape.next_tape_loc - parser.dom_parser.doc->tape.get());
simdjson_really_inline uint32_t next_tape_index(structural_parser &iter) {
return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get());
}
simdjson_really_inline void empty_container(structural_parser &parser, internal::tape_type start, internal::tape_type end) {
auto start_index = next_tape_index(parser);
simdjson_really_inline void empty_container(structural_parser &iter, internal::tape_type start, internal::tape_type end) {
auto start_index = next_tape_index(iter);
tape.append(start_index+2, start);
tape.append(start_index, end);
}
simdjson_really_inline void start_container(structural_parser &parser) {
parser.dom_parser.open_containers[parser.depth].tape_index = next_tape_index(parser);
parser.dom_parser.open_containers[parser.depth].count = 0;
simdjson_really_inline void start_container(structural_parser &iter) {
iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter);
iter.dom_parser.open_containers[iter.depth].count = 0;
tape.skip(); // We don't actually *write* the start element until the end.
}
simdjson_really_inline void end_container(structural_parser &parser, internal::tape_type start, internal::tape_type end) noexcept {
simdjson_really_inline void end_container(structural_parser &iter, internal::tape_type start, internal::tape_type end) noexcept {
// Write the ending tape element, pointing at the start location
const uint32_t start_tape_index = parser.dom_parser.open_containers[parser.depth].tape_index;
const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index;
tape.append(start_tape_index, end);
// Write the start tape element, pointing at the end location (and including count)
// count can overflow if it exceeds 24 bits... so we saturate
// the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff).
const uint32_t count = parser.dom_parser.open_containers[parser.depth].count;
const uint32_t count = iter.dom_parser.open_containers[iter.depth].count;
const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count;
tape_writer::write(parser.dom_parser.doc->tape[start_tape_index], next_tape_index(parser) | (uint64_t(cntsat) << 32), start);
tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start);
}
simdjson_really_inline uint8_t *on_start_string(structural_parser &parser) noexcept {
simdjson_really_inline uint8_t *on_start_string(structural_parser &iter) noexcept {
// we advance the point, accounting for the fact that we have a NULL termination
tape.append(current_string_buf_loc - parser.dom_parser.doc->string_buf.get(), internal::tape_type::STRING);
tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING);
return current_string_buf_loc + sizeof(uint32_t);
}