From 779ce184fb7ce02dd1a46928ba508eebb45ee535 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Tue, 18 Dec 2018 14:21:22 -0500 Subject: [PATCH] Getting ready to document the tape format. --- README.md | 2 +- include/simdjson/numberparsing.h | 2 +- include/simdjson/parsedjson.h | 20 ++++++++++++++------ jsonchecker/fail40_s64boverflow.json | 1 + jsonchecker/pass11.json | 1 + src/stage34_unified.cpp | 21 ++++++++++++++++----- 6 files changed, 34 insertions(+), 13 deletions(-) create mode 100644 jsonchecker/fail40_s64boverflow.json create mode 100644 jsonchecker/pass11.json diff --git a/README.md b/README.md index d4e0ef2e7..d71b0bfad 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ To simplify the engineering, we make some assumptions. ## Features - The input string is unmodified. (Parsers like sajson and RapidJSON use the input string as a buffer.) -- We parse integers and floating-point numbers as separate types which allows us to support large 64-bit integers in [-9223372036854775808,9223372036854775808). Among the parsers that differentiate between integers and floating-point numbers, not all support 64-bit integers. (For example, sajson stores integers larger than 2147483648 as floating-point numbers.) +- We parse integers and floating-point numbers as separate types which allows us to support large 64-bit integers in [-9223372036854775808,9223372036854775808), like a Java `long` or a C/C++ `long long`. Among the parsers that differentiate between integers and floating-point numbers, not all support 64-bit integers. (For example, sajson rejects JSON files with integers larger than or equal to 2147483648. RapidJSON will parse a file containing an overly long integer like 18446744073709551616 as a floating-point number) When we cannot represent exactly an integer as a signed 64-bit value, we reject the JSON document. - We do full UTF-8 validation as part of the parsing. (Parsers like fastjson, gason and dropbox json11 do not do UTF-8 validation.) - We fully validate the numbers. (Parsers like gason and ultranjson will accept `[0e+]` as valid JSON.) - We validate string content for unescaped characters. (Parsers like fastjson and ultrajson accept unescaped line breaks and tags in strings.) diff --git a/include/simdjson/numberparsing.h b/include/simdjson/numberparsing.h index 3338edf72..d367fb968 100644 --- a/include/simdjson/numberparsing.h +++ b/include/simdjson/numberparsing.h @@ -483,7 +483,7 @@ static really_inline bool parse_number(const u8 *const buf, #endif } } else { - if (unlikely(digitcount >= 19)) { // this is uncommon!!! + if (unlikely(digitcount >= 18)) { // this is uncommon!!! return parse_large_integer(buf, pj, offset, found_minus); } diff --git a/include/simdjson/parsedjson.h b/include/simdjson/parsedjson.h index 939339081..173e74ea2 100644 --- a/include/simdjson/parsedjson.h +++ b/include/simdjson/parsedjson.h @@ -217,8 +217,10 @@ public: bool dump_raw_tape(std::ostream &os) { if(!isvalid) return false; size_t tapeidx = 0; - u64 tape_val = tape[tapeidx++]; + u64 tape_val = tape[tapeidx]; u8 type = (tape_val >> 56); + os << tapeidx << " : " << type; + tapeidx++; size_t howmany = 0; if (type == 'r') { howmany = tape_val & JSONVALUEMASK; @@ -226,10 +228,12 @@ public: printf("Error: no starting root node?"); return false; } + os << "\t// pointing to " << howmany <<" (right after last node)\n"; + u64 payload; for (; tapeidx < howmany; tapeidx++) { os << tapeidx << " : "; tape_val = tape[tapeidx]; - u64 payload = tape_val & JSONVALUEMASK; + payload = tape_val & JSONVALUEMASK; type = (tape_val >> 56); switch (type) { case '"': // we have a string @@ -261,16 +265,16 @@ public: os << "false\n"; break; case '{': // we have an object - os << "{\t// pointing to next tape location " << payload << "\n"; + os << "{\t// pointing to next tape location " << payload << " (first node after the scope) \n"; break; case '}': // we end an object - os << "}\t// pointing to previous tape location " << payload << "\n"; + os << "}\t// pointing to previous tape location " << payload << " (start of the scope) \n"; break; case '[': // we start an array - os << "[\t// pointing to next tape location " << payload << "\n"; + os << "[\t// pointing to next tape location " << payload << " (first node after the scope) \n"; break; case ']': // we end an array - os << "]\t// pointing to previous tape location " << payload << "\n"; + os << "]\t// pointing to previous tape location " << payload << " (start of the scope) \n"; break; case 'r': // we start and end with the root node printf("end of root\n"); @@ -279,6 +283,10 @@ public: return false; } } + tape_val = tape[tapeidx]; + payload = tape_val & JSONVALUEMASK; + type = (tape_val >> 56); + os << tapeidx << " : "<< type <<"\t// pointing to " << payload <<" (start root)\n"; return true; } diff --git a/jsonchecker/fail40_s64boverflow.json b/jsonchecker/fail40_s64boverflow.json new file mode 100644 index 000000000..b0898f38f --- /dev/null +++ b/jsonchecker/fail40_s64boverflow.json @@ -0,0 +1 @@ +9223372036854775808 diff --git a/jsonchecker/pass11.json b/jsonchecker/pass11.json new file mode 100644 index 000000000..aef5454e6 --- /dev/null +++ b/jsonchecker/pass11.json @@ -0,0 +1 @@ +4611686018427387904 diff --git a/src/stage34_unified.cpp b/src/stage34_unified.cpp index 8d56da83a..f8abda6eb 100644 --- a/src/stage34_unified.cpp +++ b/src/stage34_unified.cpp @@ -103,12 +103,25 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) { if (depth > pj.depthcapacity) { goto fail; } + UPDATE_CHAR(); switch (c) { case '{': + pj.containing_scope_offset[depth] = pj.get_current_loc(); + pj.ret_address[depth] = &&start_continue; + depth++; + if (depth > pj.depthcapacity) { + goto fail; + } pj.write_tape(0, c); // strangely, moving this to object_begin slows things down goto object_begin; case '[': + pj.containing_scope_offset[depth] = pj.get_current_loc(); + pj.ret_address[depth] = &&start_continue; + depth++; + if (depth > pj.depthcapacity) { + goto fail; + } pj.write_tape(0, c); goto array_begin; #define SIMDJSON_ALLOWANYTHINGINROOT @@ -216,11 +229,6 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) { default: goto fail; } -#ifdef SIMDJSON_ALLOWANYTHINGINROOT - depth--; // for fall-through cases (e.g., documents containing just a string) - pj.annotate_previousloc(pj.containing_scope_offset[depth], - pj.get_current_loc()); -#endif // ALLOWANYTHINGINROOT start_continue: DEBUG_PRINTF("in start_object_close\n"); // the string might not be NULL terminated. @@ -465,6 +473,7 @@ array_continue: succeed: DEBUG_PRINTF("in succeed, depth = %d \n", depth); + depth --; if(depth != 0) { printf("internal bug\n"); abort(); @@ -473,6 +482,8 @@ succeed: printf("internal bug\n"); abort(); } + pj.annotate_previousloc(pj.containing_scope_offset[depth], + pj.get_current_loc()); pj.write_tape(pj.containing_scope_offset[depth], 'r'); // r is root