diff --git a/benchmark/minifiercompetition.cpp b/benchmark/minifiercompetition.cpp index 55a731088..441f4c50d 100644 --- a/benchmark/minifiercompetition.cpp +++ b/benchmark/minifiercompetition.cpp @@ -113,11 +113,11 @@ int main(int argc, char *argv[]) { BEST_TIME("sajson despaced", sajson::parse(sajson::bounded_allocation(ast_buffer, astbuffersize), sajson::mutable_string_view(minisize, buffer)).is_valid(), true, memcpy(buffer, minibuffer, p.second), repeat, volume, true); - ParsedJson *pj_ptr = allocate_ParsedJson(p.second); + ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024); ParsedJson &pj(*pj_ptr); BEST_TIME("json_parse orig", json_parse((const u8*)buffer, p.second, pj), true, memcpy(buffer, p.first, p.second), repeat, volume, true); - ParsedJson *pj_ptr2 = allocate_ParsedJson(p.second); + ParsedJson *pj_ptr2 = allocate_ParsedJson(p.second, 1024); ParsedJson &pj2(*pj_ptr2); diff --git a/benchmark/parse.cpp b/benchmark/parse.cpp index b1f4d7e2c..d874bbfdb 100644 --- a/benchmark/parse.cpp +++ b/benchmark/parse.cpp @@ -126,7 +126,7 @@ int main(int argc, char *argv[]) { if(verbose) cout << "[verbose] loading " << filename << endl; pair p = get_corpus(filename); if(verbose) cout << "[verbose] loaded " << filename << " ("<< p.second << " bytes)" << endl; - ParsedJson *pj_ptr = allocate_ParsedJson(p.second); + ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024); ParsedJson &pj(*pj_ptr); if(verbose) cout << "[verbose] allocated memory for parsed JSON " << endl; diff --git a/benchmark/parsingcompetition.cpp b/benchmark/parsingcompetition.cpp index b1ca42a3e..c9922224a 100644 --- a/benchmark/parsingcompetition.cpp +++ b/benchmark/parsingcompetition.cpp @@ -73,7 +73,7 @@ int main(int argc, char *argv[]) { std::cout << p.second << " B "; std::cout << std::endl; } - ParsedJson *pj_ptr = allocate_ParsedJson(p.second); + ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024); if (pj_ptr == NULL) { std::cerr << "can't allocate memory" << std::endl; return EXIT_FAILURE; diff --git a/include/jsonparser/jsonparser.h b/include/jsonparser/jsonparser.h index a47622b6e..a1e00abc9 100644 --- a/include/jsonparser/jsonparser.h +++ b/include/jsonparser/jsonparser.h @@ -12,7 +12,7 @@ // Return NULL if memory cannot be allocated. // This structure is meant to be reused from document to document, as needed. // you can use deallocate_ParsedJson to deallocate the memory. -ParsedJson *allocate_ParsedJson(size_t len); +ParsedJson *allocate_ParsedJson(size_t len, size_t maxdepth); // deallocate a ParsedJson struct (see allocate_ParsedJson) void deallocate_ParsedJson(ParsedJson *pj_ptr); diff --git a/include/jsonparser/numberparsing.h b/include/jsonparser/numberparsing.h index 151c256ef..39d71bae0 100644 --- a/include/jsonparser/numberparsing.h +++ b/include/jsonparser/numberparsing.h @@ -237,7 +237,7 @@ parse_highprecision_float(const u8 *const buf, UNUSED size_t len, exponent += (negexp ? -expnumber : expnumber); } if (i == 0) { - pj.write_tape_double(depth, 0.0); + pj.write_tape_double(0.0); #ifdef JSON_TEST_NUMBERS // for unit testing foundFloat(0.0, buf + offset); #endif @@ -252,7 +252,7 @@ parse_highprecision_float(const u8 *const buf, UNUSED size_t len, double d = i; d *= power_of_ten[308 + exponent]; d = negative ? -d : d; - pj.write_tape_double(depth, d); + pj.write_tape_double(d); #ifdef JSON_TEST_NUMBERS // for unit testing foundFloat(d, buf + offset); #endif @@ -325,7 +325,7 @@ static never_inline bool parse_large_integer(const u8 *const buf, } } int64_t signed_answer = negative ? -i : i; - pj.write_tape_s64(depth, signed_answer); + pj.write_tape_s64(signed_answer); #ifdef JSON_TEST_NUMBERS // for unit testing foundInteger(signed_answer, buf + offset); #endif @@ -468,7 +468,7 @@ static really_inline bool parse_number(const u8 *const buf, UNUSED size_t len, // We want 0.1e1 to be a float. ////////// if (i == 0) { - pj.write_tape_double(depth, 0.0); + pj.write_tape_double(0.0); #ifdef JSON_TEST_NUMBERS // for unit testing foundFloat(0.0, buf + offset); #endif @@ -483,7 +483,7 @@ static really_inline bool parse_number(const u8 *const buf, UNUSED size_t len, double d = i; d *= power_of_ten[308 + exponent]; // d = negative ? -d : d; - pj.write_tape_double(depth, d); + pj.write_tape_double(d); #ifdef JSON_TEST_NUMBERS // for unit testing foundFloat(d, buf + offset); #endif @@ -493,7 +493,7 @@ static really_inline bool parse_number(const u8 *const buf, UNUSED size_t len, return parse_large_integer(buf, len, pj, depth, offset, found_zero, found_minus); } - pj.write_tape_s64(depth, i); + pj.write_tape_s64(i); #ifdef JSON_TEST_NUMBERS // for unit testing foundInteger(i, buf + offset); #endif diff --git a/include/jsonparser/simdjson_internal.h b/include/jsonparser/simdjson_internal.h index b7ddf2eca..d8af44655 100644 --- a/include/jsonparser/simdjson_internal.h +++ b/include/jsonparser/simdjson_internal.h @@ -32,13 +32,18 @@ public: size_t bytecapacity; // indicates how many bits are meant to be supported by // structurals size_t depthcapacity; // how deep we can go + + u32 current_loc; u8 *structurals; u32 n_structural_indexes; u32 *structural_indexes; u64 * tape;//[MAX_TAPE]; - u32 * tape_locs; + u32 * containing_scope_offset; + void * * ret_address; + u8 * string_buf;// should be at least bytecapacity + u8 *current_string_buf_loc; u8 * number_buf;// holds either doubles or longs, really // should be at least 4 * bytecapacity u8 *current_number_buf_loc; @@ -46,6 +51,7 @@ public: void init() { current_string_buf_loc = string_buf; current_number_buf_loc = number_buf; + current_loc = 0; //for (u32 i = 0; i < MAX_DEPTH; i++) { // tape_locs[i] = i * MAX_TAPE_ENTRIES; @@ -75,32 +81,52 @@ public: } }*/ } - // TODO: will need a way of saving strings that's a bit more encapsulated - void write_tape(u32 depth, u64 val, u8 c) { - tape[tape_locs[depth]] = val | (((u64)c) << 56); - tape_locs[depth]++; + // all elements are stored on the tape using a 64-bit word. + // + // strings, double and ints are stored as + // a 64-bit word with a pointer to the actual value + // + // + // + // for objects or arrays, store [ or { at the beginning and } and ] at the end. + // For the openings ([ or {), we annotate them with a reference to the location on the tape of + // the end, and for then closings (} and ]), we annotate them with a reference to the + // location of the opening + // + // + + // this should be considered a private function + void write_tape(u64 val, u8 c) { + tape[current_loc++] = val | (((u64)c) << 56); + //tape[tape_locs[depth]] = val | (((u64)c) << 56); + //tape_locs[depth]++; } - void write_tape_s64(u32 depth, s64 i) { + + void write_tape_s64(s64 i) { *((s64 *)current_number_buf_loc) = i; current_number_buf_loc += 8; - write_tape(depth, current_number_buf_loc - number_buf, 'l'); + write_tape(current_number_buf_loc - number_buf, 'l'); } - void write_tape_double(u32 depth, double d) { + void write_tape_double(double d) { *((double *)current_number_buf_loc) = d; current_number_buf_loc += 8; - write_tape(depth, current_number_buf_loc - number_buf, 'd'); + write_tape(current_number_buf_loc - number_buf, 'd'); } - u32 save_loc(u32 depth) { - return tape_locs[depth]; + u32 get_current_loc() { + return current_loc; } - void write_saved_loc(u32 saved_loc, u64 val, u8 c) { + void annotate_previousloc(u32 saved_loc,u64 val) { + tape[saved_loc] |= val; + } + + /*void write_saved_loc(u32 saved_loc, u64 val, u8 c) { tape[saved_loc] = val | (((u64)c) << 56); - } + }*/ // public interface #if 1 @@ -121,13 +147,13 @@ public: bool prev(); // valid if we're not at the start of a scope bool up(); // valid if we are at depth != 0 bool down(); // valid if we're at a [ or { call site; moves us to header of that scope - void to_start_scope(); // move us to the start of our current scope; always succeeds - void to_end_scope(); // move us to the start of our current scope; always succeeds + //void to_start_scope(); // move us to the start of our current scope; always succeeds + //void to_end_scope(); // move us to the start of our current scope; always succeeds // these navigation elements move us across scope if need be, so allow us to iterate over // everything at a given depth - bool next_flat(); // valid if we're not at the end of a tape - bool prev_flat(); // valid if we're not at the start of a tape + //bool next_flat(); // valid if we're not at the end of a tape + //bool prev_flat(); // valid if we're not at the start of a tape void print(std::ostream & os); // print the thing we're currently pointing at u8 get_type(); // retrieve the character code of what we're looking at: [{"sltfn are the possibilities diff --git a/include/jsonparser/stringparsing.h b/include/jsonparser/stringparsing.h index 9a3f9b8df..093f96176 100644 --- a/include/jsonparser/stringparsing.h +++ b/include/jsonparser/stringparsing.h @@ -114,7 +114,7 @@ really_inline bool parse_string(const u8 *buf, UNUSED size_t len, // we encountered quotes first. Move dst to point to quotes and exit dst[quote_dist] = 0; // null terminate and get out - pj.write_tape(depth, pj.current_string_buf_loc - pj.string_buf, '"'); + pj.write_tape(pj.current_string_buf_loc - pj.string_buf, '"'); pj.current_string_buf_loc = dst + quote_dist + 1; // the +1 is due to the 0 value #ifdef CHECKUNESCAPED diff --git a/src/jsonioutil.cpp b/src/jsonioutil.cpp index ff443fb0e..31d7426f6 100644 --- a/src/jsonioutil.cpp +++ b/src/jsonioutil.cpp @@ -23,6 +23,7 @@ std::pair get_corpus(std::string filename) { size_t length = buffer.str().size(); // +1 for null u8* aligned_buffer = (u8 *)allocate_aligned_buffer(length); memcpy(aligned_buffer, buffer.str().c_str(), length); + aligned_buffer[length] = '\0'; is.close(); return std::make_pair((u8 *)aligned_buffer, length); } diff --git a/src/jsonparser.cpp b/src/jsonparser.cpp index 289772db8..1fed5baff 100644 --- a/src/jsonparser.cpp +++ b/src/jsonparser.cpp @@ -5,7 +5,11 @@ // returns NULL if memory cannot be allocated // This structure is meant to be reused from document to document, as needed. // you can use deallocate_ParsedJson to deallocate the memory. -ParsedJson *allocate_ParsedJson(size_t len) { +ParsedJson *allocate_ParsedJson(size_t len, size_t maxdepth) { + if((maxdepth == 0) || (len == 0)) { + std::cerr << "capacities must be non-zero " << std::endl; + return NULL; + } ParsedJson *pj_ptr = new ParsedJson; if (pj_ptr == NULL) { std::cerr << "Could not allocate memory for core struct." << std::endl; @@ -32,13 +36,15 @@ ParsedJson *allocate_ParsedJson(size_t len) { pj.string_buf = new u8[ROUNDUP_N(len, 64)]; pj.number_buf = new u8[4 * ROUNDUP_N(len, 64)]; pj.tape = new u64[ROUNDUP_N(len, 64)]; - size_t depthcapacity = ROUNDUP_N(len, 64); - pj.tape_locs = new u32[depthcapacity]; + pj.containing_scope_offset = new u32[maxdepth]; + pj.ret_address = new void*[maxdepth]; - if ((pj.string_buf == NULL) || (pj.number_buf == NULL) || (pj.tape == NULL) || (pj.tape_locs == NULL)) { + if ((pj.string_buf == NULL) || (pj.number_buf == NULL) || (pj.tape == NULL) + || (pj.containing_scope_offset == NULL) || (pj.ret_address == NULL) ) { std::cerr << "Could not allocate memory" << std::endl; - delete[] pj.tape_locs; + delete[] pj.ret_address; + delete[] pj.containing_scope_offset; delete[] pj.tape; delete[] pj.number_buf; delete[] pj.string_buf; @@ -49,14 +55,15 @@ ParsedJson *allocate_ParsedJson(size_t len) { } pj.bytecapacity = len; - pj.depthcapacity = depthcapacity; + pj.depthcapacity = maxdepth; return pj_ptr; } void deallocate_ParsedJson(ParsedJson *pj_ptr) { if (pj_ptr == NULL) return; - delete[] pj_ptr->tape_locs; + delete[] pj_ptr->ret_address; + delete[] pj_ptr->containing_scope_offset; delete[] pj_ptr->tape; delete[] pj_ptr->number_buf; delete[] pj_ptr->string_buf; diff --git a/src/stage1_find_marks.cpp b/src/stage1_find_marks.cpp index 2f6b2af08..2d18eb5e3 100644 --- a/src/stage1_find_marks.cpp +++ b/src/stage1_find_marks.cpp @@ -251,7 +251,7 @@ WARN_UNUSED *(u64 *)(pj.structurals + idx / 8) = structurals; } if(buf[len] != '\0') { - std::cerr << "Your string should NULL terminated." << std::endl; + std::cerr << "Your string should be NULL terminated." << std::endl; return false; } diff --git a/src/stage34_unified.cpp b/src/stage34_unified.cpp index 81bd23105..976c76e84 100644 --- a/src/stage34_unified.cpp +++ b/src/stage34_unified.cpp @@ -75,11 +75,11 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) { u32 i = 0; // index of the structural character (0,1,2,3...) u32 idx; // location of the structural character in the input (buf) u8 c; // used to track the (structural) character we are looking at, updated by UPDATE_CHAR macro - //u32 depth = START_DEPTH; // an arbitrary starting depth + u32 depth = 0;//START_DEPTH; // an arbitrary starting depth //void * ret_address[MAX_DEPTH]; // used to store "labels as value" (non-standard compiler extension) // a call site is the start of either an object or an array ('[' or '{') - //u32 last_loc = 0; // this is the location of the previous call site + // this is the location of the previous call site // (in the tape, at the given depth); // we only need one. @@ -105,42 +105,23 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) { // this macro reads the next structural character, updating idx, i and c. #define UPDATE_CHAR() { idx = pj.structural_indexes[i++]; c = buf[idx]; DEBUG_PRINTF("Got %c at %d (%d offset)\n", c, idx, i-1);} - // format: call site has 2 entries: 56-bit + '{' or '[' entries pointing first to header then to this location - // scope has 2 entries: 56 + '_' entries pointing first to call site then to the last entry in this scope -#define OPEN_SCOPE() { \ - pj.write_saved_loc(last_loc, pj.save_loc(depth), '_'); \ - pj.write_tape(depth, last_loc, '_'); \ - containing_scope_offset[depth] = pj.save_loc(depth); \ - pj.write_tape(depth, 0, '_'); \ - } -// we are going to increase the depth, we write on the tape the 'c' which is going to be either { or [ -#define ESTABLISH_CALLSITE(RETURN_LABEL, SITE_LABEL) { \ - pj.write_tape(depth, containing_scope_offset[depth], c); \ - last_loc = pj.save_loc(depth); \ - pj.write_tape(depth, 0, c); \ - ret_address[depth] = RETURN_LABEL; \ - depth++; \ - goto SITE_LABEL; \ - } ////////////////////////////// START STATE ///////////////////////////// - +printf("at start\n"); DEBUG_PRINTF("at start\n"); + pj.ret_address[depth] = &&start_continue; + pj.containing_scope_offset[depth] = pj.get_current_loc(); + pj.write_tape(0, 'r'); // r for root, 0 is going to get overwritten + depth++;// everything starts at depth = 1, depth = 0 is just for the root + if(depth > pj.depthcapacity) { + goto fail; + } + printf("got char %c \n",c); UPDATE_CHAR(); - // do these two speculatively as we will always do - // them except on fail, in which case it doesn't matter - ret_address[depth] = &&start_continue; - containing_scope_offset[depth] = pj.save_loc(depth); - pj.write_tape(depth, 0, c); // dummy entries - - last_loc = pj.save_loc(depth); - pj.write_tape(depth, 0, c); // dummy entries - depth++; - switch (c) { case '{': goto object_begin; case '[': goto array_begin; @@ -162,19 +143,19 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) { if (!is_valid_true_atom(buf + idx)) { goto fail; } - pj.write_tape(depth, 0, c); + pj.write_tape(0, c); goto start_continue; case 'f': if (!is_valid_false_atom(buf + idx)) { goto fail; } - pj.write_tape(depth, 0, c); + pj.write_tape(0, c); goto start_continue; case 'n': if (!is_valid_null_atom(buf + idx)) { goto fail; } - pj.write_tape(depth, 0, c); + pj.write_tape(0, c); goto start_continue; case '0': { if (!parse_number(buf, len, pj, depth, idx, true, false)) { @@ -199,7 +180,6 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) { } start_continue: - // land here after popping our outer object if an object DEBUG_PRINTF("in start_object_close\n"); UPDATE_CHAR(); switch (c) { @@ -210,8 +190,14 @@ start_continue: ////////////////////////////// OBJECT STATES ///////////////////////////// object_begin: + printf("in object_begin %c \n",c); DEBUG_PRINTF("in object_begin\n"); - OPEN_SCOPE(); + pj.containing_scope_offset[depth] = pj.get_current_loc(); + pj.write_tape(0, c); + depth ++; + if(depth > pj.depthcapacity) { + goto fail; + } UPDATE_CHAR(); switch (c) { case '"': { @@ -225,6 +211,8 @@ object_begin: } object_key_state: + printf("in object_key_state %c \n",c); + DEBUG_PRINTF("in object_key_state\n"); UPDATE_CHAR(); if (c != ':') { @@ -241,17 +229,17 @@ object_key_state: case 't': if (!is_valid_true_atom(buf + idx)) { goto fail; } - pj.write_tape(depth, 0, c); + pj.write_tape(0, c); break; case 'f': if (!is_valid_false_atom(buf + idx)) { goto fail; } - pj.write_tape(depth, 0, c); + pj.write_tape(0, c); break; case 'n': if (!is_valid_null_atom(buf + idx)) { goto fail; } - pj.write_tape(depth, 0, c); + pj.write_tape(0, c); break; case '0': { if (!parse_number(buf, len, pj, depth, idx, true, false)) { @@ -272,15 +260,19 @@ object_key_state: break; } case '{': { - ESTABLISH_CALLSITE(&&object_continue, object_begin); + pj.ret_address[depth] = &&object_continue; + goto object_begin; } case '[': { - ESTABLISH_CALLSITE(&&object_continue, array_begin); + pj.ret_address[depth] = &&object_continue; + goto array_begin; } default: goto fail; } object_continue: + printf("in object_continue %c \n",c); + DEBUG_PRINTF("in object_continue\n"); UPDATE_CHAR(); switch (c) { @@ -302,19 +294,25 @@ object_continue: scope_end: // write our tape location to the header scope - pj.write_saved_loc(containing_scope_offset[depth], pj.save_loc(depth), '_'); depth--; + pj.write_tape(pj.containing_scope_offset[depth], c); + pj.annotate_previousloc(pj.containing_scope_offset[depth], pj.get_current_loc()); // goto saved_state - goto *ret_address[depth]; + goto *pj.ret_address[depth]; ////////////////////////////// ARRAY STATES ///////////////////////////// array_begin: - DEBUG_PRINTF("in array_begin\n"); - OPEN_SCOPE(); - // fall through + printf("in array_begin %c \n",c); + DEBUG_PRINTF("in array_begin\n"); + pj.containing_scope_offset[depth] = pj.get_current_loc(); + pj.write_tape(0, c); + depth ++; + if(depth > pj.depthcapacity) { + goto fail; + } UPDATE_CHAR(); if (c == ']') { goto scope_end; @@ -333,17 +331,17 @@ main_array_switch: case 't': if (!is_valid_true_atom(buf + idx)) { goto fail; } - pj.write_tape(depth, 0, c); + pj.write_tape(0, c); break; case 'f': if (!is_valid_false_atom(buf + idx)) { goto fail; } - pj.write_tape(depth, 0, c); + pj.write_tape(0, c); break; case 'n': if (!is_valid_null_atom(buf + idx)) { goto fail; } - pj.write_tape(depth, 0, c); + pj.write_tape(0, c); break; case '0': { @@ -365,15 +363,19 @@ main_array_switch: break; } case '{': { - ESTABLISH_CALLSITE(&&array_continue, object_begin); + pj.ret_address[depth] = &&array_continue; + goto object_begin; } case '[': { - ESTABLISH_CALLSITE(&&array_continue, array_begin); + pj.ret_address[depth] = &&array_continue; + goto array_begin; } default: goto fail; } array_continue: + printf("in array_begin %c \n",c); + DEBUG_PRINTF("in array_continue\n"); UPDATE_CHAR(); switch (c) { @@ -386,6 +388,13 @@ array_continue: succeed: DEBUG_PRINTF("in succeed\n"); + // we annotate the root node + depth--; + // next line allows us to go back to the start + pj.write_tape(pj.containing_scope_offset[depth], 'r');// r is root + // next line tells the root node how to go to the end + pj.annotate_previousloc(pj.containing_scope_offset[depth], pj.get_current_loc()); + #ifdef DEBUG pj.dump_tapes(); #endif diff --git a/tests/allparserscheckfile.cpp b/tests/allparserscheckfile.cpp index 483c87568..4dc5c6c2d 100644 --- a/tests/allparserscheckfile.cpp +++ b/tests/allparserscheckfile.cpp @@ -62,7 +62,7 @@ int main(int argc, char *argv[]) { std::cout << p.second << " B "; std::cout << std::endl; } - ParsedJson *pj_ptr = allocate_ParsedJson(p.second); + ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024); if (pj_ptr == NULL) { std::cerr << "can't allocate memory" << std::endl; return EXIT_FAILURE; diff --git a/tests/jsoncheck.cpp b/tests/jsoncheck.cpp index 6c3c53864..2b9b2eecc 100644 --- a/tests/jsoncheck.cpp +++ b/tests/jsoncheck.cpp @@ -57,7 +57,7 @@ bool validate(const char *dirname) { strcpy(fullpath + dirlen, name); } std::pair p = get_corpus(fullpath); - ParsedJson *pj_ptr = allocate_ParsedJson(p.second); + ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024); if(pj_ptr == NULL) { std::cerr<< "can't allocate memory"< p = get_corpus(fullpath); // terrible hack but just to get it working - ParsedJson *pj_ptr = allocate_ParsedJson(p.second); + ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024); if (pj_ptr == NULL) { std::cerr << "can't allocate memory" << std::endl; return false; diff --git a/tests/stringparsingcheck.cpp b/tests/stringparsingcheck.cpp index c859b2e63..e7f02771c 100644 --- a/tests/stringparsingcheck.cpp +++ b/tests/stringparsingcheck.cpp @@ -327,7 +327,7 @@ bool validate(const char *dirname) { } std::pair p = get_corpus(fullpath); // terrible hack but just to get it working - ParsedJson *pj_ptr = allocate_ParsedJson(p.second); + ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024); if (pj_ptr == NULL) { std::cerr << "can't allocate memory" << std::endl; return false;