diff --git a/Makefile b/Makefile index 166a845c0..194d7a8b4 100644 --- a/Makefile +++ b/Makefile @@ -19,9 +19,9 @@ endif EXECUTABLES=parse jsoncheck numberparsingcheck stringparsingcheck minifiercompetition parsingcompetition minify allparserscheckfile -HEADERS= include/jsonparser/simdutf8check.h include/jsonparser/stringparsing.h include/jsonparser/numberparsing.h include/jsonparser/jsonparser.h include/jsonparser/common_defs.h include/jsonparser/jsonioutil.h benchmark/benchmark.h benchmark/linux/linux-perf-events.h include/jsonparser/simdjson_internal.h include/jsonparser/stage1_find_marks.h include/jsonparser/stage2_flatten.h include/jsonparser/stage34_unified.h include/jsonparser/jsoncharutils.h +HEADERS= include/simdjson/simdutf8check.h include/simdjson/stringparsing.h include/simdjson/numberparsing.h include/simdjson/jsonparser.h include/simdjson/common_defs.h include/simdjson/jsonioutil.h benchmark/benchmark.h benchmark/linux/linux-perf-events.h include/simdjson/parsedjson.h include/simdjson/stage1_find_marks.h include/simdjson/stage2_flatten.h include/simdjson/stage34_unified.h include/simdjson/jsoncharutils.h include/simdjson/jsonformatutils.h LIBFILES=src/jsonioutil.cpp src/jsonparser.cpp src/stage1_find_marks.cpp src/stage2_flatten.cpp src/stage34_unified.cpp -MINIFIERHEADERS=include/jsonparser/jsonminifier.h include/jsonparser/simdprune_tables.h +MINIFIERHEADERS=include/simdjson/jsonminifier.h include/simdjson/simdprune_tables.h MINIFIERLIBFILES=src/jsonminifier.cpp @@ -100,6 +100,10 @@ allparserscheckfile: tests/allparserscheckfile.cpp $(HEADERS) $(LIBFILES) $(OBJE parsehisto: benchmark/parse.cpp $(HEADERS) $(LIBFILES) $(CXX) $(CXXFLAGS) -o parsehisto benchmark/parse.cpp $(LIBFILES) $(LIBFLAGS) -DBUILDHISTOGRAM +cppcheck: + cppcheck --enable=all src/*.cpp benchmarks/*.cpp tests/*.cpp -Iinclude -I. -Ibenchmark/linux + + clean: rm -f $(OBJECTS) $(EXECUTABLES) $(EXTRA_EXECUTABLES) diff --git a/README.md b/README.md index 65b4fa94f..33ae156ed 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Goal: Speed up the parsing of JSON per se. ## Code example ```C -#include "jsonparser/jsonparser.h" +#include "simdjson/jsonparser.h" /... diff --git a/benchmark/minifiercompetition.cpp b/benchmark/minifiercompetition.cpp index d0a4a0f7e..f458ace77 100644 --- a/benchmark/minifiercompetition.cpp +++ b/benchmark/minifiercompetition.cpp @@ -2,9 +2,9 @@ #include #include "benchmark.h" -#include "jsonparser/jsonioutil.h" -#include "jsonparser/jsonminifier.h" -#include "jsonparser/jsonparser.h" +#include "simdjson/jsonioutil.h" +#include "simdjson/jsonminifier.h" +#include "simdjson/jsonparser.h" // #define RAPIDJSON_SSE2 // bad // #define RAPIDJSON_SSE42 // bad diff --git a/benchmark/parse.cpp b/benchmark/parse.cpp index 15bd9fa5b..d7513ea26 100644 --- a/benchmark/parse.cpp +++ b/benchmark/parse.cpp @@ -1,34 +1,37 @@ -#include "jsonparser/common_defs.h" -#include "linux-perf-events.h" -#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include + +#include #include #include -#include #include -#include #include #include #include #include #include -#include -#include -#include -#include #include -#include #include -#include -#include + + +#include "linux-perf-events.h" //#define DEBUG -#include "jsonparser/jsonparser.h" -#include "jsonparser/jsonioutil.h" -#include "jsonparser/simdjson_internal.h" -#include "jsonparser/stage1_find_marks.h" -#include "jsonparser/stage2_flatten.h" -#include "jsonparser/stage34_unified.h" +#include "simdjson/common_defs.h" +#include "simdjson/jsonparser.h" +#include "simdjson/jsonioutil.h" +#include "simdjson/parsedjson.h" +#include "simdjson/stage1_find_marks.h" +#include "simdjson/stage2_flatten.h" +#include "simdjson/stage34_unified.h" using namespace std; int main(int argc, char *argv[]) { diff --git a/benchmark/parsingcompetition.cpp b/benchmark/parsingcompetition.cpp index 02bf1b0aa..021c91596 100644 --- a/benchmark/parsingcompetition.cpp +++ b/benchmark/parsingcompetition.cpp @@ -1,14 +1,15 @@ #include -#include "jsonparser/jsonparser.h" +#include "simdjson/jsonparser.h" #include "benchmark.h" -// #define RAPIDJSON_SSE2 // bad -// #define RAPIDJSON_SSE42 // bad +// #define RAPIDJSON_SSE2 // bad for performance +// #define RAPIDJSON_SSE42 // bad for performance #include "rapidjson/document.h" -#include "rapidjson/reader.h" // you have to check in the submodule +#include "rapidjson/reader.h" #include "rapidjson/stringbuffer.h" #include "rapidjson/writer.h" + #include "json11.cpp" #include "sajson.h" #include "fastjson.cpp" @@ -18,7 +19,6 @@ extern "C" { #include "ultrajsondec.c" #include "ujdecode.h" - } using namespace rapidjson; using namespace std; diff --git a/include/jsonparser/common_defs.h b/include/simdjson/common_defs.h similarity index 100% rename from include/jsonparser/common_defs.h rename to include/simdjson/common_defs.h diff --git a/include/jsonparser/jsoncharutils.h b/include/simdjson/jsoncharutils.h similarity index 98% rename from include/jsonparser/jsoncharutils.h rename to include/simdjson/jsoncharutils.h index 3eb529b78..e4d821930 100644 --- a/include/jsonparser/jsoncharutils.h +++ b/include/simdjson/jsoncharutils.h @@ -1,7 +1,7 @@ #pragma once -#include "jsonparser/common_defs.h" -#include "jsonparser/simdjson_internal.h" +#include "simdjson/common_defs.h" +#include "simdjson/parsedjson.h" // structural chars here are // they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c @@ -109,4 +109,3 @@ inline size_t codepoint_to_utf8(uint32_t cp, u8 *c) { return 0; // bad r } - diff --git a/include/simdjson/jsonformatutils.h b/include/simdjson/jsonformatutils.h new file mode 100644 index 000000000..13386ec7d --- /dev/null +++ b/include/simdjson/jsonformatutils.h @@ -0,0 +1,33 @@ +#pragma once + +#include + +static inline void print_with_escapes(const unsigned char *src) { + while (*src) { + switch (*src) { + case '\n': + putchar('\\'); + putchar('n'); + break; + case '\"': + putchar('\\'); + putchar('"'); + break; + case '\t': + putchar('\\'); + putchar('t'); + break; + case '\\': + putchar('\\'); + putchar('\\'); + break; + default: + if (*src <= 0x1F) { + printf("\\u%x", *src); + } else + putchar(*src); + } + src++; + } +} + diff --git a/include/jsonparser/jsonioutil.h b/include/simdjson/jsonioutil.h similarity index 96% rename from include/jsonparser/jsonioutil.h rename to include/simdjson/jsonioutil.h index 62289fba0..ca734f80e 100644 --- a/include/jsonparser/jsonioutil.h +++ b/include/simdjson/jsonioutil.h @@ -7,7 +7,7 @@ #include #include -#include "common_defs.h" +#include "simdjson/common_defs.h" // if you must provide a pointer to some data, create it with this function: // length is the max. size in bytes of the string diff --git a/include/jsonparser/jsonminifier.h b/include/simdjson/jsonminifier.h similarity index 100% rename from include/jsonparser/jsonminifier.h rename to include/simdjson/jsonminifier.h diff --git a/include/jsonparser/jsonparser.h b/include/simdjson/jsonparser.h similarity index 75% rename from include/jsonparser/jsonparser.h rename to include/simdjson/jsonparser.h index a1e00abc9..323cdc625 100644 --- a/include/jsonparser/jsonparser.h +++ b/include/simdjson/jsonparser.h @@ -1,11 +1,11 @@ #pragma once -#include "common_defs.h" -#include "jsonioutil.h" -#include "simdjson_internal.h" -#include "stage1_find_marks.h" -#include "stage2_flatten.h" -#include "stage34_unified.h" +#include "simdjson/common_defs.h" +#include "simdjson/jsonioutil.h" +#include "simdjson/parsedjson.h" +#include "simdjson/stage1_find_marks.h" +#include "simdjson/stage2_flatten.h" +#include "simdjson/stage34_unified.h" // Allocate a ParsedJson structure that can support document // up to len bytes. diff --git a/include/jsonparser/numberparsing.h b/include/simdjson/numberparsing.h similarity index 99% rename from include/jsonparser/numberparsing.h rename to include/simdjson/numberparsing.h index 7558f931e..d5dc81296 100644 --- a/include/jsonparser/numberparsing.h +++ b/include/simdjson/numberparsing.h @@ -1,8 +1,8 @@ #pragma once -#include "common_defs.h" -#include "jsonparser/jsoncharutils.h" -#include "jsonparser/simdjson_internal.h" +#include "simdjson/common_defs.h" +#include "simdjson/jsoncharutils.h" +#include "simdjson/parsedjson.h" static const double power_of_ten[] = { 1e-308, 1e-307, 1e-306, 1e-305, 1e-304, 1e-303, 1e-302, 1e-301, 1e-300, diff --git a/include/jsonparser/simdjson_internal.h b/include/simdjson/parsedjson.h similarity index 90% rename from include/jsonparser/simdjson_internal.h rename to include/simdjson/parsedjson.h index 0e4f4c066..2de281e2d 100644 --- a/include/jsonparser/simdjson_internal.h +++ b/include/simdjson/parsedjson.h @@ -6,51 +6,16 @@ /* Microsoft C/C++-compatible compiler */ #include #else -#include #include #endif #include #include + +#include "simdjson/jsonformatutils.h" + #define JSONVALUEMASK 0xFFFFFFFFFFFFFF; -static inline void print_with_escapes(const unsigned char *src) { - while (*src) { - switch (*src) { - case '\n': - putchar('\\'); - putchar('n'); - break; - case '\"': - putchar('\\'); - putchar('"'); - break; - case '\t': - putchar('\\'); - putchar('t'); - break; - case '\\': - putchar('\\'); - putchar('\\'); - break; - default: - if (*src <= 0x1F) { - printf("\\u%x", *src); - } else - putchar(*src); - } - src++; - } -} - -// const u32 MAX_DEPTH = 2048; -// const u32 DEPTH_SAFETY_MARGIN = 32; // should be power-of-2 as we check this -// with a modulo in our hot stage 3 loop -// const u32 START_DEPTH = DEPTH_SAFETY_MARGIN; -// const u32 REDLINE_DEPTH = MAX_DEPTH - DEPTH_SAFETY_MARGIN; -// const size_t MAX_TAPE_ENTRIES = 127 * 1024; -// const size_t MAX_TAPE = MAX_DEPTH * MAX_TAPE_ENTRIES; - ///////////// // TODO: move this to be more like a real class // currently, you need to create it like so... @@ -114,7 +79,7 @@ public: if ((inobjectidx[depth] > 0) && (type != ']')) printf(", "); inobjectidx[depth]++; - } else if (inobject) { + } else { //if (inobject) { if ((inobjectidx[depth] > 0) && ((inobjectidx[depth] & 1) == 0) && (type != '}')) printf(", "); @@ -204,6 +169,8 @@ public: really_inline void write_tape_double(double d) { write_tape(0, 'd'); + static_assert(sizeof(d) == sizeof(tape[current_loc]), + "mismatch size"); tape[current_loc++] =*( (u64*) &d); } diff --git a/include/jsonparser/simdprune_tables.h b/include/simdjson/simdprune_tables.h similarity index 100% rename from include/jsonparser/simdprune_tables.h rename to include/simdjson/simdprune_tables.h diff --git a/include/jsonparser/simdutf8check.h b/include/simdjson/simdutf8check.h similarity index 98% rename from include/jsonparser/simdutf8check.h rename to include/simdjson/simdutf8check.h index dd48ef279..5d2b19183 100644 --- a/include/jsonparser/simdutf8check.h +++ b/include/simdjson/simdutf8check.h @@ -4,7 +4,12 @@ #include #include #include +#ifdef _MSC_VER +/* Microsoft C/C++-compatible compiler */ +#include +#else #include +#endif #include /* * legal utf-8 byte sequence diff --git a/include/jsonparser/stage1_find_marks.h b/include/simdjson/stage1_find_marks.h similarity index 79% rename from include/jsonparser/stage1_find_marks.h rename to include/simdjson/stage1_find_marks.h index 11c1b86e4..a623595b2 100644 --- a/include/jsonparser/stage1_find_marks.h +++ b/include/simdjson/stage1_find_marks.h @@ -1,7 +1,7 @@ #pragma once #include "common_defs.h" -#include "simdjson_internal.h" +#include "parsedjson.h" WARN_UNUSED bool find_structural_bits(const u8 *buf, size_t len, ParsedJson &pj); diff --git a/include/jsonparser/stage2_flatten.h b/include/simdjson/stage2_flatten.h similarity index 53% rename from include/jsonparser/stage2_flatten.h rename to include/simdjson/stage2_flatten.h index bec26c293..449581b3c 100644 --- a/include/jsonparser/stage2_flatten.h +++ b/include/simdjson/stage2_flatten.h @@ -1,7 +1,7 @@ #pragma once -#include "common_defs.h" -#include "simdjson_internal.h" +#include "simdjson/common_defs.h" +#include "simdjson/parsedjson.h" WARN_UNUSED bool flatten_indexes(size_t len, ParsedJson &pj); diff --git a/include/jsonparser/stage34_unified.h b/include/simdjson/stage34_unified.h similarity index 64% rename from include/jsonparser/stage34_unified.h rename to include/simdjson/stage34_unified.h index c2432374a..21ed82af5 100644 --- a/include/jsonparser/stage34_unified.h +++ b/include/simdjson/stage34_unified.h @@ -1,7 +1,7 @@ #pragma once -#include "common_defs.h" -#include "simdjson_internal.h" +#include "simdjson/common_defs.h" +#include "simdjson/parsedjson.h" void init_state_machine(); diff --git a/include/jsonparser/stringparsing.h b/include/simdjson/stringparsing.h similarity index 98% rename from include/jsonparser/stringparsing.h rename to include/simdjson/stringparsing.h index ede34b678..1152eee88 100644 --- a/include/jsonparser/stringparsing.h +++ b/include/simdjson/stringparsing.h @@ -1,8 +1,8 @@ #pragma once -#include "common_defs.h" -#include "jsonparser/simdjson_internal.h" -#include "jsonparser/jsoncharutils.h" +#include "simdjson/common_defs.h" +#include "simdjson/parsedjson.h" +#include "simdjson/jsoncharutils.h" // begin copypasta diff --git a/src/jsonioutil.cpp b/src/jsonioutil.cpp index 31d7426f6..504bccce0 100644 --- a/src/jsonioutil.cpp +++ b/src/jsonioutil.cpp @@ -1,4 +1,4 @@ -#include "jsonparser/jsonioutil.h" +#include "simdjson/jsonioutil.h" #include diff --git a/src/jsonminifier.cpp b/src/jsonminifier.cpp index 62eef28b4..49933ecd3 100644 --- a/src/jsonminifier.cpp +++ b/src/jsonminifier.cpp @@ -67,7 +67,7 @@ size_t jsonminify(const unsigned char *bytes, size_t howmany, #include #endif // _MSC_VER -#include "jsonparser/simdprune_tables.h" +#include "simdjson/simdprune_tables.h" #include #ifndef __clang__ static inline __m256i _mm256_loadu2_m128i(__m128i const *__addr_hi, diff --git a/src/jsonparser.cpp b/src/jsonparser.cpp index 40ee0160b..70eaf43ba 100644 --- a/src/jsonparser.cpp +++ b/src/jsonparser.cpp @@ -1,4 +1,4 @@ -#include "jsonparser/jsonparser.h" +#include "simdjson/jsonparser.h" // allocate a ParsedJson structure that can support document // up to len bytes. diff --git a/src/stage1_find_marks.cpp b/src/stage1_find_marks.cpp index 2d18eb5e3..7ec38b3fc 100644 --- a/src/stage1_find_marks.cpp +++ b/src/stage1_find_marks.cpp @@ -2,14 +2,13 @@ /* Microsoft C/C++-compatible compiler */ #include #else -#include #include #endif #include -#include "jsonparser/common_defs.h" -#include "jsonparser/simdjson_internal.h" +#include "simdjson/common_defs.h" +#include "simdjson/parsedjson.h" #define UTF8VALIDATE // It seems that many parsers do UTF-8 validation. @@ -17,7 +16,7 @@ // allows it. It appears that sajson might do utf-8 // validation #ifdef UTF8VALIDATE -#include "jsonparser/simdutf8check.h" +#include "simdjson/simdutf8check.h" #endif using namespace std; @@ -156,7 +155,7 @@ WARN_UNUSED u64 quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128( _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFF), 0)); quote_mask ^= prev_iter_inside_quote; - prev_iter_inside_quote = (u64)((s64)quote_mask >> 63); + prev_iter_inside_quote = (u64)((s64)quote_mask >> 63); // right shift of a signed value expected to be well-defined and standard compliant as of C++20 dumpbits(quote_mask, "quote_mask"); // How do we build up a user traversable data structure diff --git a/src/stage2_flatten.cpp b/src/stage2_flatten.cpp index 02b2268be..4a1eed7ad 100644 --- a/src/stage2_flatten.cpp +++ b/src/stage2_flatten.cpp @@ -2,14 +2,13 @@ /* Microsoft C/C++-compatible compiler */ #include #else -#include #include #endif #include -#include "jsonparser/common_defs.h" -#include "jsonparser/simdjson_internal.h" +#include "simdjson/common_defs.h" +#include "simdjson/parsedjson.h" #ifndef NO_PDEP_PLEASE #define NO_PDEP_PLEASE // though this is not always a win, it seems to diff --git a/src/stage34_unified.cpp b/src/stage34_unified.cpp index 0bdfa230c..952a21f86 100644 --- a/src/stage34_unified.cpp +++ b/src/stage34_unified.cpp @@ -2,18 +2,17 @@ /* Microsoft C/C++-compatible compiler */ #include #else -#include #include #endif #include #include -#include "jsonparser/common_defs.h" -#include "jsonparser/jsoncharutils.h" -#include "jsonparser/numberparsing.h" -#include "jsonparser/simdjson_internal.h" -#include "jsonparser/stringparsing.h" +#include "simdjson/common_defs.h" +#include "simdjson/jsoncharutils.h" +#include "simdjson/numberparsing.h" +#include "simdjson/parsedjson.h" +#include "simdjson/stringparsing.h" #include //#define DEBUG diff --git a/tests/allparserscheckfile.cpp b/tests/allparserscheckfile.cpp index 660a024ea..70f8819d4 100644 --- a/tests/allparserscheckfile.cpp +++ b/tests/allparserscheckfile.cpp @@ -1,6 +1,6 @@ #include -#include "jsonparser/jsonparser.h" +#include "simdjson/jsonparser.h" // #define RAPIDJSON_SSE2 // bad diff --git a/tests/jsoncheck.cpp b/tests/jsoncheck.cpp index a9487f813..a80f125df 100644 --- a/tests/jsoncheck.cpp +++ b/tests/jsoncheck.cpp @@ -7,7 +7,7 @@ #include #include -#include "jsonparser/jsonparser.h" +#include "simdjson/jsonparser.h" /** * Does the file filename ends with the given extension. diff --git a/tests/numberparsingcheck.cpp b/tests/numberparsingcheck.cpp index 017f5ee32..bd364de4e 100644 --- a/tests/numberparsingcheck.cpp +++ b/tests/numberparsingcheck.cpp @@ -11,7 +11,7 @@ #define JSON_TEST_NUMBERS #endif -#include "jsonparser/common_defs.h" +#include "simdjson/common_defs.h" int parse_error; char *fullpath; @@ -82,7 +82,7 @@ inline void foundFloat(double result, const u8 *buf) { } } -#include "jsonparser/jsonparser.h" +#include "simdjson/jsonparser.h" #include "src/stage34_unified.cpp" /** diff --git a/tests/stringparsingcheck.cpp b/tests/stringparsingcheck.cpp index 3805ce53c..255d9ecc5 100644 --- a/tests/stringparsingcheck.cpp +++ b/tests/stringparsingcheck.cpp @@ -11,7 +11,7 @@ #define JSON_TEST_STRINGS #endif -#include "jsonparser/common_defs.h" +#include "simdjson/common_defs.h" char *fullpath; @@ -281,7 +281,7 @@ inline void foundString(const u8 *buf, const u8 *parsed_begin, } } -#include "jsonparser/jsonparser.h" +#include "simdjson/jsonparser.h" #include "src/stage34_unified.cpp" /** diff --git a/tools/minify.cpp b/tools/minify.cpp index 89c9cf6a6..9c9449929 100644 --- a/tools/minify.cpp +++ b/tools/minify.cpp @@ -1,7 +1,7 @@ #include -#include "jsonparser/jsonioutil.h" -#include "jsonparser/jsonminifier.h" +#include "simdjson/jsonioutil.h" +#include "simdjson/jsonminifier.h" int main(int argc, char *argv[]) { if (argc != 2) {