diff --git a/LIMITATIONS.md b/LIMITATIONS.md new file mode 100644 index 000000000..bffaabf1d --- /dev/null +++ b/LIMITATIONS.md @@ -0,0 +1,4 @@ +To simplify the engineering, we make some assumptions that can be lifted with some effort: + +- This library cannot parse JSON document of size 16MB or more. +- We expect the input memory pointer to 256-bit aligned and to be padded (e.g., with spaces) so that it can be read entirely in blocks of 256 bits. diff --git a/Makefile b/Makefile index 9d8480819..6edaf7902 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ .PHONY: clean cleandist -CXXFLAGS = -std=c++11 -O2 -march=native -Wall -Wextra -Wshadow -Iinclude -Ibenchmark/linux -Idependencies/double-conversion -Idependencies/rapidjson/include -Ldependencies/double-conversion/release +CXXFLAGS = -std=c++11 -g2 -O2 -march=native -Wall -Wextra -Wshadow -Iinclude -Ibenchmark/linux -Idependencies/double-conversion -Idependencies/rapidjson/include -Ldependencies/double-conversion/release LIBFLAGS = -ldouble-conversion EXECUTABLES=parse jsoncheck minifiercompetition parsingcompetition diff --git a/include/jsonparser/jsonioutil.h b/include/jsonparser/jsonioutil.h index 67ba72a90..54804b7bc 100644 --- a/include/jsonparser/jsonioutil.h +++ b/include/jsonparser/jsonioutil.h @@ -9,8 +9,12 @@ #include "common_defs.h" +// load a file in memory... // get a corpus; pad out to cache line so we can always use SIMD // throws exceptions in case of failure +// first element of the pair is a string (null terminated) +// whereas the second element is the length. +// caller is responsible to free (free std::pair.first) std::pair get_corpus(std::string filename); #endif diff --git a/include/jsonparser/simdjson_internal.h b/include/jsonparser/simdjson_internal.h index 997222507..d36721bbc 100644 --- a/include/jsonparser/simdjson_internal.h +++ b/include/jsonparser/simdjson_internal.h @@ -9,6 +9,7 @@ #endif #include +#define MAX_JSON_BYTES 0xFFFFFF const u32 MAX_DEPTH = 256; const u32 DEPTH_SAFETY_MARGIN = 32; // should be power-of-2 as we check this @@ -28,9 +29,9 @@ struct ParsedJson { // grossly overprovisioned u64 tape[MAX_TAPE]; u32 tape_locs[MAX_DEPTH]; - u8 string_buf[512 * 1024]; + u8 string_buf[MAX_JSON_BYTES]; u8 *current_string_buf_loc; - u8 number_buf[512 * 1024]; // holds either doubles or longs, really + u8 number_buf[MAX_JSON_BYTES * 4]; // holds either doubles or longs, really u8 *current_number_buf_loc; }; diff --git a/src/jsonioutil.cpp b/src/jsonioutil.cpp index 581afe8e9..4ac8a9d9e 100644 --- a/src/jsonioutil.cpp +++ b/src/jsonioutil.cpp @@ -5,13 +5,16 @@ std::pair get_corpus(std::string filename) { if (is) { std::stringstream buffer; buffer << is.rdbuf(); - size_t length = buffer.str().size(); + size_t length = buffer.str().size(); // +1 for null char *aligned_buffer; - if (posix_memalign((void **)&aligned_buffer, 64, ROUNDUP_N(length, 64))) { + size_t paddedlength = ROUNDUP_N(length, 64); + if (posix_memalign((void **)&aligned_buffer, 64, paddedlength + 1)) { throw std::runtime_error("Could not allocate sufficient memory"); }; - memset(aligned_buffer, 0x20, ROUNDUP_N(length, 64)); + //memset(aligned_buffer, 0x20, ROUNDUP_N(length + 1, 64)); memcpy(aligned_buffer, buffer.str().c_str(), length); + memset(aligned_buffer + length, 0x20, paddedlength - length); + aligned_buffer[paddedlength] = '\0'; is.close(); return std::make_pair((u8 *)aligned_buffer, length); } diff --git a/src/jsonparser.cpp b/src/jsonparser.cpp index ccedadcbe..4e00f14ef 100644 --- a/src/jsonparser.cpp +++ b/src/jsonparser.cpp @@ -6,8 +6,8 @@ // This structure is meant to be reused from document to document, as needed. // you can use deallocate_ParsedJson to deallocate the memory. ParsedJson *allocate_ParsedJson(size_t len) { - if (len > 0xffffff) { - std::cerr << "Currently only support JSON files < 16MB, requested length: " + if (len > MAX_JSON_BYTES) { + std::cerr << "Currently only support JSON files having up to "<structural_indexes; - delete[] pj_ptr->structurals; + free(pj_ptr->structurals); delete pj_ptr; }