admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Still working

Browse Source
This commit is contained in:
Daniel Lemire
2018-11-27 10:10:39 -05:00
parent 50defa510f
commit 5fae7b2100
15 changed files with 134 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmark/minifiercompetition.cpp
+2 -2
View File
@@ -113,11 +113,11 @@ int main(int argc, char *argv[]) {
BEST_TIME("sajson despaced", sajson::parse(sajson::bounded_allocation(ast_buffer, astbuffersize), sajson::mutable_string_view(minisize, buffer)).is_valid(), true, memcpy(buffer, minibuffer, p.second), repeat, volume, true);
ParsedJson *pj_ptr = allocate_ParsedJson(p.second);
ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024);
ParsedJson &pj(*pj_ptr);
BEST_TIME("json_parse orig", json_parse((const u8*)buffer, p.second, pj), true, memcpy(buffer, p.first, p.second), repeat, volume, true);
ParsedJson *pj_ptr2 = allocate_ParsedJson(p.second);
ParsedJson *pj_ptr2 = allocate_ParsedJson(p.second, 1024);
ParsedJson &pj2(*pj_ptr2);
benchmark/parse.cpp
+1 -1
View File
@@ -126,7 +126,7 @@ int main(int argc, char *argv[]) {
if(verbose) cout << "[verbose] loading " << filename << endl;
pair<u8 *, size_t> p = get_corpus(filename);
if(verbose) cout << "[verbose] loaded " << filename << " ("<< p.second << " bytes)" << endl;
ParsedJson *pj_ptr = allocate_ParsedJson(p.second);
ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024);
ParsedJson &pj(*pj_ptr);
if(verbose) cout << "[verbose] allocated memory for parsed JSON " << endl;
benchmark/parsingcompetition.cpp
+1 -1
View File
@@ -73,7 +73,7 @@ int main(int argc, char *argv[]) {
std::cout << p.second << " B ";
std::cout << std::endl;
}
ParsedJson *pj_ptr = allocate_ParsedJson(p.second);
ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024);
if (pj_ptr == NULL) {
std::cerr << "can't allocate memory" << std::endl;
return EXIT_FAILURE;
include/jsonparser/jsonparser.h
+1 -1
View File
@@ -12,7 +12,7 @@
// Return NULL if memory cannot be allocated.
// This structure is meant to be reused from document to document, as needed.
// you can use deallocate_ParsedJson to deallocate the memory.
ParsedJson *allocate_ParsedJson(size_t len);
ParsedJson *allocate_ParsedJson(size_t len, size_t maxdepth);
// deallocate a ParsedJson struct (see allocate_ParsedJson)
void deallocate_ParsedJson(ParsedJson *pj_ptr);
include/jsonparser/numberparsing.h
+6 -6
View File
@@ -237,7 +237,7 @@ parse_highprecision_float(const u8 *const buf, UNUSED size_t len,
exponent += (negexp ? -expnumber : expnumber);
}
if (i == 0) {
pj.write_tape_double(depth, 0.0);
pj.write_tape_double(0.0);
#ifdef JSON_TEST_NUMBERS // for unit testing
foundFloat(0.0, buf + offset);
#endif
@@ -252,7 +252,7 @@ parse_highprecision_float(const u8 *const buf, UNUSED size_t len,
double d = i;
d *= power_of_ten[308 + exponent];
d = negative ? -d : d;
pj.write_tape_double(depth, d);
pj.write_tape_double(d);
#ifdef JSON_TEST_NUMBERS // for unit testing
foundFloat(d, buf + offset);
#endif
@@ -325,7 +325,7 @@ static never_inline bool parse_large_integer(const u8 *const buf,
}
}
int64_t signed_answer = negative ? -i : i;
pj.write_tape_s64(depth, signed_answer);
pj.write_tape_s64(signed_answer);
#ifdef JSON_TEST_NUMBERS // for unit testing
foundInteger(signed_answer, buf + offset);
#endif
@@ -468,7 +468,7 @@ static really_inline bool parse_number(const u8 *const buf, UNUSED size_t len,
// We want 0.1e1 to be a float.
//////////
if (i == 0) {
pj.write_tape_double(depth, 0.0);
pj.write_tape_double(0.0);
#ifdef JSON_TEST_NUMBERS // for unit testing
foundFloat(0.0, buf + offset);
#endif
@@ -483,7 +483,7 @@ static really_inline bool parse_number(const u8 *const buf, UNUSED size_t len,
double d = i;
d *= power_of_ten[308 + exponent];
// d = negative ? -d : d;
pj.write_tape_double(depth, d);
pj.write_tape_double(d);
#ifdef JSON_TEST_NUMBERS // for unit testing
foundFloat(d, buf + offset);
#endif
@@ -493,7 +493,7 @@ static really_inline bool parse_number(const u8 *const buf, UNUSED size_t len,
return parse_large_integer(buf, len, pj, depth, offset, found_zero,
found_minus);
}
pj.write_tape_s64(depth, i);
pj.write_tape_s64(i);
#ifdef JSON_TEST_NUMBERS // for unit testing
foundInteger(i, buf + offset);
#endif
include/jsonparser/simdjson_internal.h
+43 -17
View File
@@ -32,13 +32,18 @@ public:
size_t bytecapacity; // indicates how many bits are meant to be supported by
// structurals
size_t depthcapacity; // how deep we can go
u32 current_loc;
u8 *structurals;
u32 n_structural_indexes;
u32 *structural_indexes;
u64 * tape;//[MAX_TAPE];
u32 * tape_locs;
u32 * containing_scope_offset;
void * * ret_address;
u8 * string_buf;// should be at least bytecapacity
u8 *current_string_buf_loc;
u8 * number_buf;// holds either doubles or longs, really // should be at least 4 * bytecapacity
u8 *current_number_buf_loc;
@@ -46,6 +51,7 @@ public:
void init() {
current_string_buf_loc = string_buf;
current_number_buf_loc = number_buf;
current_loc = 0;
//for (u32 i = 0; i < MAX_DEPTH; i++) {
// tape_locs[i] = i * MAX_TAPE_ENTRIES;
@@ -75,32 +81,52 @@ public:
}
}*/
}
// TODO: will need a way of saving strings that's a bit more encapsulated
void write_tape(u32 depth, u64 val, u8 c) {
tape[tape_locs[depth]] = val | (((u64)c) << 56);
tape_locs[depth]++;
// all elements are stored on the tape using a 64-bit word.
//
// strings, double and ints are stored as
// a 64-bit word with a pointer to the actual value
//
//
//
// for objects or arrays, store [ or { at the beginning and } and ] at the end.
// For the openings ([ or {), we annotate them with a reference to the location on the tape of
// the end, and for then closings (} and ]), we annotate them with a reference to the
// location of the opening
//
//
// this should be considered a private function
void write_tape(u64 val, u8 c) {
tape[current_loc++] = val | (((u64)c) << 56);
//tape[tape_locs[depth]] = val | (((u64)c) << 56);
//tape_locs[depth]++;
}
void write_tape_s64(u32 depth, s64 i) {
void write_tape_s64(s64 i) {
*((s64 *)current_number_buf_loc) = i;
current_number_buf_loc += 8;
write_tape(depth, current_number_buf_loc - number_buf, 'l');
write_tape(current_number_buf_loc - number_buf, 'l');
}
void write_tape_double(u32 depth, double d) {
void write_tape_double(double d) {
*((double *)current_number_buf_loc) = d;
current_number_buf_loc += 8;
write_tape(depth, current_number_buf_loc - number_buf, 'd');
write_tape(current_number_buf_loc - number_buf, 'd');
}
u32 save_loc(u32 depth) {
return tape_locs[depth];
u32 get_current_loc() {
return current_loc;
}
void write_saved_loc(u32 saved_loc, u64 val, u8 c) {
void annotate_previousloc(u32 saved_loc,u64 val) {
tape[saved_loc] |= val;
}
/*void write_saved_loc(u32 saved_loc, u64 val, u8 c) {
tape[saved_loc] = val | (((u64)c) << 56);
}
}*/
// public interface
#if 1
@@ -121,13 +147,13 @@ public:
bool prev(); // valid if we're not at the start of a scope
bool up(); // valid if we are at depth != 0
bool down(); // valid if we're at a [ or { call site; moves us to header of that scope
void to_start_scope(); // move us to the start of our current scope; always succeeds
void to_end_scope(); // move us to the start of our current scope; always succeeds
//void to_start_scope(); // move us to the start of our current scope; always succeeds
//void to_end_scope(); // move us to the start of our current scope; always succeeds
// these navigation elements move us across scope if need be, so allow us to iterate over
// everything at a given depth
bool next_flat(); // valid if we're not at the end of a tape
bool prev_flat(); // valid if we're not at the start of a tape
//bool next_flat(); // valid if we're not at the end of a tape
//bool prev_flat(); // valid if we're not at the start of a tape
void print(std::ostream & os); // print the thing we're currently pointing at
u8 get_type(); // retrieve the character code of what we're looking at: [{"sltfn are the possibilities
include/jsonparser/stringparsing.h
+1 -1
View File
@@ -114,7 +114,7 @@ really_inline bool parse_string(const u8 *buf, UNUSED size_t len,
// we encountered quotes first. Move dst to point to quotes and exit
dst[quote_dist] = 0; // null terminate and get out
pj.write_tape(depth, pj.current_string_buf_loc - pj.string_buf, '"');
pj.write_tape(pj.current_string_buf_loc - pj.string_buf, '"');
pj.current_string_buf_loc = dst + quote_dist + 1; // the +1 is due to the 0 value
#ifdef CHECKUNESCAPED
src/jsonioutil.cpp
+1
View File
@@ -23,6 +23,7 @@ std::pair<u8 *, size_t> get_corpus(std::string filename) {
size_t length = buffer.str().size(); // +1 for null
u8* aligned_buffer = (u8 *)allocate_aligned_buffer(length);
memcpy(aligned_buffer, buffer.str().c_str(), length);
aligned_buffer[length] = '\0';
is.close();
return std::make_pair((u8 *)aligned_buffer, length);
}
src/jsonparser.cpp
+14 -7
View File
@@ -5,7 +5,11 @@
// returns NULL if memory cannot be allocated
// This structure is meant to be reused from document to document, as needed.
// you can use deallocate_ParsedJson to deallocate the memory.
ParsedJson *allocate_ParsedJson(size_t len) {
ParsedJson *allocate_ParsedJson(size_t len, size_t maxdepth) {
if((maxdepth == 0) || (len == 0)) {
std::cerr << "capacities must be non-zero " << std::endl;
return NULL;
}
ParsedJson *pj_ptr = new ParsedJson;
if (pj_ptr == NULL) {
std::cerr << "Could not allocate memory for core struct." << std::endl;
@@ -32,13 +36,15 @@ ParsedJson *allocate_ParsedJson(size_t len) {
pj.string_buf = new u8[ROUNDUP_N(len, 64)];
pj.number_buf = new u8[4 * ROUNDUP_N(len, 64)];
pj.tape = new u64[ROUNDUP_N(len, 64)];
size_t depthcapacity = ROUNDUP_N(len, 64);
pj.tape_locs = new u32[depthcapacity];
pj.containing_scope_offset = new u32[maxdepth];
pj.ret_address = new void*[maxdepth];
if ((pj.string_buf == NULL) || (pj.number_buf == NULL) || (pj.tape == NULL) || (pj.tape_locs == NULL)) {
if ((pj.string_buf == NULL) || (pj.number_buf == NULL) || (pj.tape == NULL)
|| (pj.containing_scope_offset == NULL) || (pj.ret_address == NULL) ) {
std::cerr << "Could not allocate memory"
<< std::endl;
delete[] pj.tape_locs;
delete[] pj.ret_address;
delete[] pj.containing_scope_offset;
delete[] pj.tape;
delete[] pj.number_buf;
delete[] pj.string_buf;
@@ -49,14 +55,15 @@ ParsedJson *allocate_ParsedJson(size_t len) {
}
pj.bytecapacity = len;
pj.depthcapacity = depthcapacity;
pj.depthcapacity = maxdepth;
return pj_ptr;
}
void deallocate_ParsedJson(ParsedJson *pj_ptr) {
if (pj_ptr == NULL)
return;
delete[] pj_ptr->tape_locs;
delete[] pj_ptr->ret_address;
delete[] pj_ptr->containing_scope_offset;
delete[] pj_ptr->tape;
delete[] pj_ptr->number_buf;
delete[] pj_ptr->string_buf;
src/stage1_find_marks.cpp
+1 -1
View File
@@ -251,7 +251,7 @@ WARN_UNUSED
*(u64 *)(pj.structurals + idx / 8) = structurals;
}
if(buf[len] != '\0') {
std::cerr << "Your string should NULL terminated." << std::endl;
std::cerr << "Your string should be NULL terminated." << std::endl;
return false;
}
src/stage34_unified.cpp
+59 -50
View File
@@ -75,11 +75,11 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) {
u32 i = 0; // index of the structural character (0,1,2,3...)
u32 idx; // location of the structural character in the input (buf)
u8 c; // used to track the (structural) character we are looking at, updated by UPDATE_CHAR macro
//u32 depth = START_DEPTH; // an arbitrary starting depth
u32 depth = 0;//START_DEPTH; // an arbitrary starting depth
//void * ret_address[MAX_DEPTH]; // used to store "labels as value" (non-standard compiler extension)
// a call site is the start of either an object or an array ('[' or '{')
//u32 last_loc = 0; // this is the location of the previous call site
// this is the location of the previous call site
// (in the tape, at the given depth);
// we only need one.
@@ -105,42 +105,23 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) {
// this macro reads the next structural character, updating idx, i and c.
#define UPDATE_CHAR() { idx = pj.structural_indexes[i++]; c = buf[idx]; DEBUG_PRINTF("Got %c at %d (%d offset)\n", c, idx, i-1);}
// format: call site has 2 entries: 56-bit + '{' or '[' entries pointing first to header then to this location
// scope has 2 entries: 56 + '_' entries pointing first to call site then to the last entry in this scope
#define OPEN_SCOPE() { \
pj.write_saved_loc(last_loc, pj.save_loc(depth), '_'); \
pj.write_tape(depth, last_loc, '_'); \
containing_scope_offset[depth] = pj.save_loc(depth); \
pj.write_tape(depth, 0, '_'); \
}
// we are going to increase the depth, we write on the tape the 'c' which is going to be either { or [
#define ESTABLISH_CALLSITE(RETURN_LABEL, SITE_LABEL) { \
pj.write_tape(depth, containing_scope_offset[depth], c); \
last_loc = pj.save_loc(depth); \
pj.write_tape(depth, 0, c); \
ret_address[depth] = RETURN_LABEL; \
depth++; \
goto SITE_LABEL; \
}
////////////////////////////// START STATE /////////////////////////////
printf("at start\n");
DEBUG_PRINTF("at start\n");
pj.ret_address[depth] = &&start_continue;
pj.containing_scope_offset[depth] = pj.get_current_loc();
pj.write_tape(0, 'r'); // r for root, 0 is going to get overwritten
depth++;// everything starts at depth = 1, depth = 0 is just for the root
if(depth > pj.depthcapacity) {
goto fail;
}
printf("got char %c \n",c);
UPDATE_CHAR();
// do these two speculatively as we will always do
// them except on fail, in which case it doesn't matter
ret_address[depth] = &&start_continue;
containing_scope_offset[depth] = pj.save_loc(depth);
pj.write_tape(depth, 0, c); // dummy entries
last_loc = pj.save_loc(depth);
pj.write_tape(depth, 0, c); // dummy entries
depth++;
switch (c) {
case '{': goto object_begin;
case '[': goto array_begin;
@@ -162,19 +143,19 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) {
if (!is_valid_true_atom(buf + idx)) {
goto fail;
}
pj.write_tape(depth, 0, c);
pj.write_tape(0, c);
goto start_continue;
case 'f':
if (!is_valid_false_atom(buf + idx)) {
goto fail;
}
pj.write_tape(depth, 0, c);
pj.write_tape(0, c);
goto start_continue;
case 'n':
if (!is_valid_null_atom(buf + idx)) {
goto fail;
}
pj.write_tape(depth, 0, c);
pj.write_tape(0, c);
goto start_continue;
case '0': {
if (!parse_number(buf, len, pj, depth, idx, true, false)) {
@@ -199,7 +180,6 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) {
}
start_continue:
// land here after popping our outer object if an object
DEBUG_PRINTF("in start_object_close\n");
UPDATE_CHAR();
switch (c) {
@@ -210,8 +190,14 @@ start_continue:
////////////////////////////// OBJECT STATES /////////////////////////////
object_begin:
printf("in object_begin %c \n",c);
DEBUG_PRINTF("in object_begin\n");
OPEN_SCOPE();
pj.containing_scope_offset[depth] = pj.get_current_loc();
pj.write_tape(0, c);
depth ++;
if(depth > pj.depthcapacity) {
goto fail;
}
UPDATE_CHAR();
switch (c) {
case '"': {
@@ -225,6 +211,8 @@ object_begin:
}
object_key_state:
printf("in object_key_state %c \n",c);
DEBUG_PRINTF("in object_key_state\n");
UPDATE_CHAR();
if (c != ':') {
@@ -241,17 +229,17 @@ object_key_state:
case 't': if (!is_valid_true_atom(buf + idx)) {
goto fail;
}
pj.write_tape(depth, 0, c);
pj.write_tape(0, c);
break;
case 'f': if (!is_valid_false_atom(buf + idx)) {
goto fail;
}
pj.write_tape(depth, 0, c);
pj.write_tape(0, c);
break;
case 'n': if (!is_valid_null_atom(buf + idx)) {
goto fail;
}
pj.write_tape(depth, 0, c);
pj.write_tape(0, c);
break;
case '0': {
if (!parse_number(buf, len, pj, depth, idx, true, false)) {
@@ -272,15 +260,19 @@ object_key_state:
break;
}
case '{': {
ESTABLISH_CALLSITE(&&object_continue, object_begin);
pj.ret_address[depth] = &&object_continue;
goto object_begin;
}
case '[': {
ESTABLISH_CALLSITE(&&object_continue, array_begin);
pj.ret_address[depth] = &&object_continue;
goto array_begin;
}
default: goto fail;
}
object_continue:
printf("in object_continue %c \n",c);
DEBUG_PRINTF("in object_continue\n");
UPDATE_CHAR();
switch (c) {
@@ -302,19 +294,25 @@ object_continue:
scope_end:
// write our tape location to the header scope
pj.write_saved_loc(containing_scope_offset[depth], pj.save_loc(depth), '_');
depth--;
pj.write_tape(pj.containing_scope_offset[depth], c);
pj.annotate_previousloc(pj.containing_scope_offset[depth], pj.get_current_loc());
// goto saved_state
goto *ret_address[depth];
goto *pj.ret_address[depth];
////////////////////////////// ARRAY STATES /////////////////////////////
array_begin:
DEBUG_PRINTF("in array_begin\n");
OPEN_SCOPE();
// fall through
printf("in array_begin %c \n",c);
DEBUG_PRINTF("in array_begin\n");
pj.containing_scope_offset[depth] = pj.get_current_loc();
pj.write_tape(0, c);
depth ++;
if(depth > pj.depthcapacity) {
goto fail;
}
UPDATE_CHAR();
if (c == ']') {
goto scope_end;
@@ -333,17 +331,17 @@ main_array_switch:
case 't': if (!is_valid_true_atom(buf + idx)) {
goto fail;
}
pj.write_tape(depth, 0, c);
pj.write_tape(0, c);
break;
case 'f': if (!is_valid_false_atom(buf + idx)) {
goto fail;
}
pj.write_tape(depth, 0, c);
pj.write_tape(0, c);
break;
case 'n': if (!is_valid_null_atom(buf + idx)) {
goto fail;
}
pj.write_tape(depth, 0, c);
pj.write_tape(0, c);
break;
case '0': {
@@ -365,15 +363,19 @@ main_array_switch:
break;
}
case '{': {
ESTABLISH_CALLSITE(&&array_continue, object_begin);
pj.ret_address[depth] = &&array_continue;
goto object_begin;
}
case '[': {
ESTABLISH_CALLSITE(&&array_continue, array_begin);
pj.ret_address[depth] = &&array_continue;
goto array_begin;
}
default: goto fail;
}
array_continue:
printf("in array_begin %c \n",c);
DEBUG_PRINTF("in array_continue\n");
UPDATE_CHAR();
switch (c) {
@@ -386,6 +388,13 @@ array_continue:
succeed:
DEBUG_PRINTF("in succeed\n");
// we annotate the root node
depth--;
// next line allows us to go back to the start
pj.write_tape(pj.containing_scope_offset[depth], 'r');// r is root
// next line tells the root node how to go to the end
pj.annotate_previousloc(pj.containing_scope_offset[depth], pj.get_current_loc());
#ifdef DEBUG
pj.dump_tapes();
#endif
tests/allparserscheckfile.cpp
+1 -1
View File
@@ -62,7 +62,7 @@ int main(int argc, char *argv[]) {
std::cout << p.second << " B ";
std::cout << std::endl;
}
ParsedJson *pj_ptr = allocate_ParsedJson(p.second);
ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024);
if (pj_ptr == NULL) {
std::cerr << "can't allocate memory" << std::endl;
return EXIT_FAILURE;
tests/jsoncheck.cpp
+1 -1
View File
@@ -57,7 +57,7 @@ bool validate(const char *dirname) {
strcpy(fullpath + dirlen, name);
}
std::pair<u8 *, size_t> p = get_corpus(fullpath);
ParsedJson *pj_ptr = allocate_ParsedJson(p.second);
ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024);
if(pj_ptr == NULL) {
std::cerr<< "can't allocate memory"<<std::endl;
return false;
tests/numberparsingcheck.cpp
+1 -1
View File
@@ -125,7 +125,7 @@ bool validate(const char *dirname) {
}
std::pair<u8 *, size_t> p = get_corpus(fullpath);
// terrible hack but just to get it working
ParsedJson *pj_ptr = allocate_ParsedJson(p.second);
ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024);
if (pj_ptr == NULL) {
std::cerr << "can't allocate memory" << std::endl;
return false;
tests/stringparsingcheck.cpp
+1 -1
View File
@@ -327,7 +327,7 @@ bool validate(const char *dirname) {
}
std::pair<u8 *, size_t> p = get_corpus(fullpath);
// terrible hack but just to get it working
ParsedJson *pj_ptr = allocate_ParsedJson(p.second);
ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024);
if (pj_ptr == NULL) {
std::cerr << "can't allocate memory" << std::endl;
return false;