diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 06d542c03..35fb3b30c 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -19,5 +19,6 @@ Reini Urban Tom Dyson Ihor Dotsenko Alexey Milovidov +Chang Liu # if you have contributed to the project and your name does not # appear in this list, please let us know! diff --git a/benchmark/distinctuseridcompetition.cpp b/benchmark/distinctuseridcompetition.cpp index d62217a7d..d1e1bd0ab 100644 --- a/benchmark/distinctuseridcompetition.cpp +++ b/benchmark/distinctuseridcompetition.cpp @@ -30,7 +30,7 @@ void print_vec(const std::vector &v) { std::cout << std::endl; } -void simdjson_scan(std::vector &answer, ParsedJson::iterator &i) { +void simdjson_scan(std::vector &answer, simdjson::ParsedJson::iterator &i) { while(i.move_forward()) { if(i.get_scope_type() == '{') { bool founduser = (i.get_string_length() == 4) && (memcmp(i.get_string(), "user", 4) == 0); @@ -48,30 +48,30 @@ void simdjson_scan(std::vector &answer, ParsedJson::iterator &i) { } __attribute__ ((noinline)) -std::vector simdjson_justdom(ParsedJson &pj) { +std::vector simdjson_justdom(simdjson::ParsedJson &pj) { std::vector answer; - ParsedJson::iterator i(pj); + simdjson::ParsedJson::iterator i(pj); simdjson_scan(answer,i); remove_duplicates(answer); return answer; } __attribute__ ((noinline)) -std::vector simdjson_computestats(const padded_string &p) { +std::vector simdjson_computestats(const simdjson::padded_string &p) { std::vector answer; - ParsedJson pj = build_parsed_json(p); + simdjson::ParsedJson pj = simdjson::build_parsed_json(p); if (!pj.isValid()) { return answer; } - ParsedJson::iterator i(pj); + simdjson::ParsedJson::iterator i(pj); simdjson_scan(answer,i); remove_duplicates(answer); return answer; } __attribute__ ((noinline)) -bool simdjson_justparse(const padded_string &p) { - ParsedJson pj = build_parsed_json(p); +bool simdjson_justparse(const simdjson::padded_string &p) { + simdjson::ParsedJson pj = simdjson::build_parsed_json(p); bool answer = !pj.isValid(); return answer; } @@ -135,7 +135,7 @@ std::vector sasjon_justdom(sajson::document & d) { } __attribute__ ((noinline)) -std::vector sasjon_computestats(const padded_string &p) { +std::vector sasjon_computestats(const simdjson::padded_string &p) { std::vector answer; char *buffer = (char *)malloc(p.size()); memcpy(buffer, p.data(), p.size()); @@ -152,7 +152,7 @@ std::vector sasjon_computestats(const padded_string &p) { } __attribute__ ((noinline)) -bool sasjon_justparse(const padded_string &p) { +bool sasjon_justparse(const simdjson::padded_string &p) { char *buffer = (char *)malloc(p.size()); memcpy(buffer, p.data(), p.size()); auto d = sajson::parse(sajson::dynamic_allocation(), @@ -210,7 +210,7 @@ std::vector rapid_justdom(rapidjson::Document &d) { } __attribute__ ((noinline)) -std::vector rapid_computestats(const padded_string &p) { +std::vector rapid_computestats(const simdjson::padded_string &p) { std::vector answer; char *buffer = (char *)malloc(p.size() + 1); memcpy(buffer, p.data(), p.size()); @@ -228,7 +228,7 @@ std::vector rapid_computestats(const padded_string &p) { } __attribute__ ((noinline)) -bool rapid_justparse(const padded_string &p) { +bool rapid_justparse(const simdjson::padded_string &p) { char *buffer = (char *)malloc(p.size() + 1); memcpy(buffer, p.data(), p.size()); buffer[p.size()] = '\0'; @@ -267,9 +267,9 @@ int main(int argc, char *argv[]) { if (optind + 1 < argc) { std::cerr << "warning: ignoring everything after " << argv[optind + 1] << std::endl; } - padded_string p; + simdjson::padded_string p; try { - get_corpus(filename).swap(p); + simdjson::get_corpus(filename).swap(p); } catch (const std::exception &e) { // caught by reference to base std::cout << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; @@ -321,7 +321,7 @@ int main(int argc, char *argv[]) { !justdata); BEST_TIME("sasjon (just parse) ", sasjon_justparse(p), false, , repeat, volume, !justdata); - ParsedJson dsimdjson = build_parsed_json(p); + simdjson::ParsedJson dsimdjson = simdjson::build_parsed_json(p); BEST_TIME("simdjson (just dom) ", simdjson_justdom(dsimdjson).size(), size, , repeat, volume, !justdata); char *buffer = (char *)malloc(p.size()); diff --git a/benchmark/minifiercompetition.cpp b/benchmark/minifiercompetition.cpp index 14d73a88d..78d681c00 100644 --- a/benchmark/minifiercompetition.cpp +++ b/benchmark/minifiercompetition.cpp @@ -65,9 +65,9 @@ int main(int argc, char *argv[]) { exit(1); } const char * filename = argv[optind]; - padded_string p; + simdjson::padded_string p; try { - get_corpus(filename).swap(p); + simdjson::get_corpus(filename).swap(p); } catch (const std::exception& e) { // caught by reference to base std::cout << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; diff --git a/benchmark/parse.cpp b/benchmark/parse.cpp index 132de5496..deaa80886 100644 --- a/benchmark/parse.cpp +++ b/benchmark/parse.cpp @@ -78,9 +78,9 @@ int main(int argc, char *argv[]) { if (verbose) { std::cout << "[verbose] loading " << filename << std::endl; } - padded_string p; + simdjson::padded_string p; try { - get_corpus(filename).swap(p); + simdjson::get_corpus(filename).swap(p); } catch (const std::exception &e) { // caught by reference to base std::cout << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; @@ -128,7 +128,7 @@ int main(int argc, char *argv[]) { std::cout << "[verbose] iteration # " << i << std::endl; } unified.start(); - ParsedJson pj; + simdjson::ParsedJson pj; bool allocok = pj.allocateCapacity(p.size()); if (!allocok) { std::cerr << "failed to allocate memory" << std::endl; @@ -145,7 +145,7 @@ int main(int argc, char *argv[]) { } unified.start(); // The default template is simdjson::instruction_set::native. - isok = (find_structural_bits<>(p.data(), p.size(), pj) == simdjson::SUCCESS); + isok = (simdjson::find_structural_bits<>(p.data(), p.size(), pj) == simdjson::SUCCESS); unified.end(results); cy1 += results[0]; cl1 += results[1]; @@ -175,7 +175,7 @@ int main(int argc, char *argv[]) { if (verbose) { std::cout << "[verbose] iteration # " << i << std::endl; } - ParsedJson pj; + simdjson::ParsedJson pj; bool allocok = pj.allocateCapacity(p.size()); if (!allocok) { std::cerr << "failed to allocate memory" << std::endl; @@ -187,7 +187,7 @@ int main(int argc, char *argv[]) { auto start = std::chrono::steady_clock::now(); // The default template is simdjson::instruction_set::native. - isok = (find_structural_bits<>(p.data(), p.size(), pj) == simdjson::SUCCESS); + isok = (simdjson::find_structural_bits<>(p.data(), p.size(), pj) == simdjson::SUCCESS); isok = isok && (simdjson::SUCCESS == unified_machine(p.data(), p.size(), pj)); auto end = std::chrono::steady_clock::now(); std::chrono::duration secs = end - start; @@ -198,7 +198,7 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE; } } - ParsedJson pj = build_parsed_json(p); // do the parsing again to get the stats + simdjson::ParsedJson pj = build_parsed_json(p); // do the parsing again to get the stats if (!pj.isValid()) { std::cerr << pj.getErrorMsg() << std::endl; std::cerr << "Could not parse. " << std::endl; diff --git a/benchmark/parseandstatcompetition.cpp b/benchmark/parseandstatcompetition.cpp index 04ba142b6..138d3d9d2 100644 --- a/benchmark/parseandstatcompetition.cpp +++ b/benchmark/parseandstatcompetition.cpp @@ -146,7 +146,7 @@ void sajson_traverse(stat_t &stats, const sajson::value &node) { } __attribute__ ((noinline)) -stat_t sasjon_computestats(const padded_string &p) { +stat_t sasjon_computestats(const simdjson::padded_string &p) { stat_t answer; char *buffer = (char *)malloc(p.size()); memcpy(buffer, p.data(), p.size()); @@ -204,7 +204,7 @@ void rapid_traverse(stat_t &stats, const rapidjson::Value &v) { } __attribute__ ((noinline)) -stat_t rapid_computestats(const padded_string &p) { +stat_t rapid_computestats(const simdjson::padded_string &p) { stat_t answer; char *buffer = (char *)malloc(p.size() + 1); memcpy(buffer, p.data(), p.size()); @@ -253,9 +253,9 @@ int main(int argc, char *argv[]) { if (optind + 1 < argc) { std::cerr << "warning: ignoring everything after " << argv[optind + 1] << std::endl; } - padded_string p; + simdjson::padded_string p; try { - get_corpus(filename).swap(p); + simdjson::get_corpus(filename).swap(p); } catch (const std::exception &e) { // caught by reference to base std::cout << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; diff --git a/benchmark/parsingcompetition.cpp b/benchmark/parsingcompetition.cpp index da9a8b64b..172972325 100644 --- a/benchmark/parsingcompetition.cpp +++ b/benchmark/parsingcompetition.cpp @@ -83,9 +83,9 @@ int main(int argc, char *argv[]) { std::cerr << "warning: ignoring everything after " << argv[optind + 1] << std::endl; } - padded_string p; + simdjson::padded_string p; try { - get_corpus(filename).swap(p); + simdjson::get_corpus(filename).swap(p); } catch (const std::exception &e) { // caught by reference to base std::cout << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; @@ -101,7 +101,7 @@ int main(int argc, char *argv[]) { std::cout << p.size() << " B "; std::cout << std::endl; } - ParsedJson pj; + simdjson::ParsedJson pj; bool allocok = pj.allocateCapacity(p.size(), 1024); if (!allocok) { diff --git a/benchmark/statisticalmodel.cpp b/benchmark/statisticalmodel.cpp index c2c06b832..3d6a1b1c0 100644 --- a/benchmark/statisticalmodel.cpp +++ b/benchmark/statisticalmodel.cpp @@ -42,9 +42,9 @@ struct stat_s { using stat_t = struct stat_s; -stat_t simdjson_computestats(const padded_string &p) { +stat_t simdjson_computestats(const simdjson::padded_string &p) { stat_t answer; - ParsedJson pj = build_parsed_json(p); + simdjson::ParsedJson pj = simdjson::build_parsed_json(p); answer.valid = pj.isValid(); if (!answer.valid) { return answer; @@ -134,9 +134,9 @@ int main(int argc, char *argv[]) { std::cerr << "warning: ignoring everything after " << argv[optind + 1] << std::endl; } - padded_string p; + simdjson::padded_string p; try { - get_corpus(filename).swap(p); + simdjson::get_corpus(filename).swap(p); } catch (const std::exception &e) { // caught by reference to base std::cerr << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; @@ -163,7 +163,7 @@ int main(int argc, char *argv[]) { s.object_count, s.array_count, s.null_count, s.true_count, s.false_count, s.byte_count, s.structural_indexes_count); #ifdef __linux__ - ParsedJson pj; + simdjson::ParsedJson pj; bool allocok = pj.allocateCapacity(p.size()); if (!allocok) { std::cerr << "failed to allocate memory" << std::endl; @@ -181,7 +181,7 @@ int main(int argc, char *argv[]) { for (uint32_t i = 0; i < iterations; i++) { unified.start(); // The default template is simdjson::instruction_set::native. - bool isok = (find_structural_bits<>(p.data(), p.size(), pj) == simdjson::SUCCESS); + bool isok = (simdjson::find_structural_bits<>(p.data(), p.size(), pj) == simdjson::SUCCESS); unified.end(results); cy1 += results[0]; diff --git a/include/simdjson/jsoncharutils.h b/include/simdjson/jsoncharutils.h index 5948c7d76..261f0e9e2 100644 --- a/include/simdjson/jsoncharutils.h +++ b/include/simdjson/jsoncharutils.h @@ -4,6 +4,7 @@ #include "simdjson/common_defs.h" #include "simdjson/parsedjson.h" +namespace simdjson { // structural chars here are // they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c (and NULL) // we are also interested in the four whitespace characters @@ -293,5 +294,6 @@ inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { // will return 0 when the code point was too large. return 0; // bad r } +} #endif diff --git a/include/simdjson/jsonformatutils.h b/include/simdjson/jsonformatutils.h index 4373f1082..8b86ec1c3 100644 --- a/include/simdjson/jsonformatutils.h +++ b/include/simdjson/jsonformatutils.h @@ -5,6 +5,7 @@ #include #include +namespace simdjson { // ends with zero char static inline void print_with_escapes(const unsigned char *src) { while (*src) { @@ -195,6 +196,7 @@ static inline void print_with_escapes(const char *src, std::ostream &os, size_t len) { print_with_escapes(reinterpret_cast(src), os, len); } +} # #endif diff --git a/include/simdjson/jsonioutil.h b/include/simdjson/jsonioutil.h index e6c93fb03..407718d01 100644 --- a/include/simdjson/jsonioutil.h +++ b/include/simdjson/jsonioutil.h @@ -12,7 +12,7 @@ #include "simdjson/padded_string.h" - +namespace simdjson { // load a file in memory... // get a corpus; pad out to cache line so we can always use SIMD @@ -29,6 +29,6 @@ // std::cout << "Could not load the file " << filename << std::endl; // } padded_string get_corpus(const std::string& filename); - +} #endif diff --git a/include/simdjson/jsonminifier.h b/include/simdjson/jsonminifier.h index 0a4852479..a588338c8 100644 --- a/include/simdjson/jsonminifier.h +++ b/include/simdjson/jsonminifier.h @@ -4,6 +4,7 @@ #include #include +namespace simdjson { // Take input from buf and remove useless whitespace, write it to out; buf and // out can be the same pointer. Result is null terminated, // return the string length (minus the null termination). @@ -22,5 +23,5 @@ static inline size_t jsonminify(const std::string_view & p, char *out) { static inline size_t jsonminify(const padded_string & p, char *out) { return jsonminify(p.data(), p.size(), out); } - +} #endif diff --git a/include/simdjson/jsonparser.h b/include/simdjson/jsonparser.h index 03844660a..a113077d9 100644 --- a/include/simdjson/jsonparser.h +++ b/include/simdjson/jsonparser.h @@ -15,6 +15,7 @@ #include #endif +namespace simdjson { // The function that users are expected to call is json_parse. // We have more than one such function because we want to support several // instruction sets. @@ -27,9 +28,9 @@ extern json_parse_functype *json_parse_ptr; // json_parse_implementation is the generic function, it is specialized for various -// SIMD instruction sets, e.g., as json_parse_implementation -// or json_parse_implementation -template +// SIMD instruction sets, e.g., as json_parse_implementation +// or json_parse_implementation +template int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) { if (pj.bytecapacity < len) { return simdjson::CAPACITY; @@ -200,7 +201,5 @@ WARN_UNUSED inline ParsedJson build_parsed_json(const padded_string &s) { return build_parsed_json(s.data(), s.length(), false); } - - - +} #endif diff --git a/include/simdjson/numberparsing.h b/include/simdjson/numberparsing.h index 76248e169..c5fca4ed8 100644 --- a/include/simdjson/numberparsing.h +++ b/include/simdjson/numberparsing.h @@ -6,6 +6,7 @@ #include "simdjson/parsedjson.h" #include "simdjson/portability.h" +namespace simdjson { // Allowable floating-point values range from std::numeric_limits::lowest() // to std::numeric_limits::max(), so from // -1.7976e308 all the way to 1.7975e308 in binary64. The lowest non-zero @@ -553,5 +554,5 @@ static really_inline bool parse_number(const uint8_t *const buf, return is_structural_or_whitespace(*p); #endif // SIMDJSON_SKIPNUMBERPARSING } - +} #endif diff --git a/include/simdjson/padded_string.h b/include/simdjson/padded_string.h index 3421e574a..6c7e7e63c 100644 --- a/include/simdjson/padded_string.h +++ b/include/simdjson/padded_string.h @@ -3,6 +3,8 @@ #include "simdjson/portability.h" #include #include + +namespace simdjson { // low-level function to allocate memory with padding so we can read passed the // "length" bytes safely. if you must provide a pointer to some data, create it // with this function: length is the max. size in bytes of the string caller is @@ -63,5 +65,6 @@ private: size_t viable_size; char *data_ptr; }; +} #endif diff --git a/include/simdjson/parsedjson.h b/include/simdjson/parsedjson.h index 9c34c7c7e..4662d653f 100644 --- a/include/simdjson/parsedjson.h +++ b/include/simdjson/parsedjson.h @@ -15,7 +15,7 @@ #define DEFAULTMAXDEPTH 1024// a JSON document with a depth exceeding 1024 is probably de facto invalid - +namespace simdjson { /************ * The JSON is parsed to a tape, see the accompanying tape.md file * for documentation. @@ -499,4 +499,5 @@ bool ParsedJson::iterator::next() { current_type = nexttype; return true; } +} #endif diff --git a/include/simdjson/portability.h b/include/simdjson/portability.h index b4eab86cb..2069cf72e 100644 --- a/include/simdjson/portability.h +++ b/include/simdjson/portability.h @@ -7,6 +7,7 @@ #include #include +namespace simdjson { static inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { return _addcarry_u64(0, value1, value2, reinterpret_cast(result)); } @@ -34,7 +35,7 @@ static inline int hamming(uint64_t input_num) { __popcnt((uint32_t)(input_num >> 32))); #endif } - +} #else #include #include @@ -42,7 +43,7 @@ static inline int hamming(uint64_t input_num) { #if defined(__BMI2__) || defined(__POPCOUNT__) || defined(__AVX2__) #include #endif - +namespace simdjson { static inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { return __builtin_uaddll_overflow(value1, value2, (unsigned long long*)result); } @@ -76,10 +77,11 @@ static inline int hamming(uint64_t input_num) { return __builtin_popcountll(input_num); #endif } - +} #endif // _MSC_VER +namespace simdjson { // portable version of posix_memalign static inline void *aligned_malloc(size_t alignment, size_t size) { void *p; @@ -139,5 +141,6 @@ static inline void aligned_free(void *memblock) { static inline void aligned_free_char(char *memblock) { aligned_free((void*)memblock); } +} #endif // SIMDJSON_PORTABILITY_H diff --git a/include/simdjson/simdjson.h b/include/simdjson/simdjson.h index 7f88d635b..9a16692df 100644 --- a/include/simdjson/simdjson.h +++ b/include/simdjson/simdjson.h @@ -3,46 +3,45 @@ #include -struct simdjson { - enum class instruction_set { - avx2, - sse4_2, - neon, - none, +namespace simdjson { +enum class instruction_set { + avx2, + sse4_2, + neon, + none, // the 'native' enum class value should point at a good default on the current machine #ifdef __AVX2__ - native = avx2 + native = avx2 #elif defined(__ARM_NEON) - native = neon + native = neon #else - // Let us assume that we have an old x64 processor, but one that has SSE (i.e., something - // that came out in the second decade of the XXIst century. - // It would be nicer to check explicitly, but there many not be a good way to do so - // that is cross-platform. - // Under Visual Studio, there is no way to check for SSE4.2 support at compile-time. - native = sse4_2 + // Let us assume that we have an old x64 processor, but one that has SSE (i.e., something + // that came out in the second decade of the XXIst century. + // It would be nicer to check explicitly, but there many not be a good way to do so + // that is cross-platform. + // Under Visual Studio, there is no way to check for SSE4.2 support at compile-time. + native = sse4_2 #endif - }; - - enum errorValues { - SUCCESS = 0, - CAPACITY, // This ParsedJson can't support a document that big - MEMALLOC, // Error allocating memory, most likely out of memory - TAPE_ERROR, // Something went wrong while writing to the tape (stage 2), this is a generic error - DEPTH_ERROR, // Your document exceeds the user-specified depth limitation - STRING_ERROR, // Problem while parsing a string - T_ATOM_ERROR, // Problem while parsing an atom starting with the letter 't' - F_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'f' - N_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'n' - NUMBER_ERROR, // Problem while parsing a number - UTF8_ERROR, // the input is not valid UTF-8 - UNITIALIZED, // unknown error, or uninitialized document - EMPTY, // no structural document found - UNESCAPED_CHARS, // found unescaped characters in a string. - UNCLOSED_STRING, // missing quote at the end - UNEXPECTED_ERROR // indicative of a bug in simdjson - }; - static const std::string& errorMsg(const int); }; +enum errorValues { + SUCCESS = 0, + CAPACITY, // This ParsedJson can't support a document that big + MEMALLOC, // Error allocating memory, most likely out of memory + TAPE_ERROR, // Something went wrong while writing to the tape (stage 2), this is a generic error + DEPTH_ERROR, // Your document exceeds the user-specified depth limitation + STRING_ERROR, // Problem while parsing a string + T_ATOM_ERROR, // Problem while parsing an atom starting with the letter 't' + F_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'f' + N_ATOM_ERROR, // Problem while parsing an atom starting with the letter 'n' + NUMBER_ERROR, // Problem while parsing a number + UTF8_ERROR, // the input is not valid UTF-8 + UNITIALIZED, // unknown error, or uninitialized document + EMPTY, // no structural document found + UNESCAPED_CHARS, // found unescaped characters in a string. + UNCLOSED_STRING, // missing quote at the end + UNEXPECTED_ERROR // indicative of a bug in simdjson +}; +const std::string& errorMsg(const int); +} #endif diff --git a/include/simdjson/simdjson_version.h b/include/simdjson/simdjson_version.h index 0a71b8952..fd072b308 100644 --- a/include/simdjson/simdjson_version.h +++ b/include/simdjson/simdjson_version.h @@ -2,9 +2,11 @@ #ifndef SIMDJSON_INCLUDE_SIMDJSON_VERSION #define SIMDJSON_INCLUDE_SIMDJSON_VERSION #define SIMDJSON_VERSION 0.1.2 +namespace simdjson { enum { SIMDJSON_VERSION_MAJOR = 0, SIMDJSON_VERSION_MINOR = 1, SIMDJSON_VERSION_REVISION = 2 }; +} #endif // SIMDJSON_INCLUDE_SIMDJSON_VERSION diff --git a/include/simdjson/simdprune_tables.h b/include/simdjson/simdprune_tables.h index 64b56c226..43566fe72 100644 --- a/include/simdjson/simdprune_tables.h +++ b/include/simdjson/simdprune_tables.h @@ -3,8 +3,9 @@ #include "simdjson/portability.h" -#ifdef __AVX__ +namespace simdjson { +#ifdef __AVX__ static const unsigned char mask128_epi8[] = { 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, @@ -34984,13 +34985,13 @@ static const unsigned char mask128_epi32[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, }; - +} #endif //__SSE3__ #ifdef __AVX2__ #include - +namespace simdjson { static const uint32_t mask256_epi32[] = { 0, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 7, 0, 2, 3, 4, 5, 6, 7, 7, 2, 3, 4, 5, 6, 7, 7, 7, 0, 1, 3, 4, 5, 6, 7, 7, 1, 3, 4, 5, 6, 7, 7, 7, 0, 3, @@ -35074,6 +35075,7 @@ static const uint32_t mask256_epi32[] = { 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; +} #endif //__AVX2__ #endif diff --git a/include/simdjson/simdutf8check.h b/include/simdjson/simdutf8check.h index 310888299..fe198991e 100644 --- a/include/simdjson/simdutf8check.h +++ b/include/simdjson/simdutf8check.h @@ -26,9 +26,8 @@ // all byte values must be no larger than 0xF4 - +namespace simdjson { #ifdef __AVX2__ - /*****************************/ static inline __m256i push_last_byte_of_a_to_b(__m256i a, __m256i b) { return _mm256_alignr_epi8(b, _mm256_permute2x128_si256(a, b, 0x21), 15); @@ -194,4 +193,5 @@ avxcheckUTF8Bytes(__m256i current_bytes, #else // __AVX2__ #warning "We require AVX2 support!" #endif // __AVX2__ +} #endif diff --git a/include/simdjson/stage1_find_marks.h b/include/simdjson/stage1_find_marks.h index 2cf0e7d98..ca4598d02 100644 --- a/include/simdjson/stage1_find_marks.h +++ b/include/simdjson/stage1_find_marks.h @@ -32,6 +32,7 @@ #define TRANSPOSE +namespace simdjson { template struct simd_input; #ifdef __AVX2__ @@ -853,5 +854,5 @@ WARN_UNUSED int find_structural_bits(const char *buf, size_t len, ParsedJson &pj) { return find_structural_bits(reinterpret_cast(buf), len, pj); } - +} #endif diff --git a/include/simdjson/stage2_build_tape.h b/include/simdjson/stage2_build_tape.h index 44890dcff..214234d1a 100644 --- a/include/simdjson/stage2_build_tape.h +++ b/include/simdjson/stage2_build_tape.h @@ -3,6 +3,7 @@ #include "simdjson/common_defs.h" +namespace simdjson { struct ParsedJson; void init_state_machine(); @@ -12,5 +13,6 @@ int unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj); WARN_UNUSED int unified_machine(const char *buf, size_t len, ParsedJson &pj); +} #endif diff --git a/include/simdjson/stringparsing.h b/include/simdjson/stringparsing.h index 1a68d0da4..8e985dcf5 100644 --- a/include/simdjson/stringparsing.h +++ b/include/simdjson/stringparsing.h @@ -5,7 +5,7 @@ #include "simdjson/jsoncharutils.h" #include "simdjson/parsedjson.h" - +namespace simdjson { // begin copypasta // These chars yield themselves: " \ / // b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab @@ -201,6 +201,6 @@ really_inline bool parse_string(UNUSED const uint8_t *buf, UNUSED size_t len, return true; #endif // SIMDJSON_SKIPSTRINGPARSING } - +} #endif diff --git a/src/jsonioutil.cpp b/src/jsonioutil.cpp index 5ac2eb47f..2e99e6e10 100644 --- a/src/jsonioutil.cpp +++ b/src/jsonioutil.cpp @@ -2,6 +2,7 @@ #include #include +namespace simdjson { char * allocate_padded_buffer(size_t length) { // we could do a simple malloc //return (char *) malloc(length + SIMDJSON_PADDING); @@ -31,3 +32,4 @@ padded_string get_corpus(const std::string& filename) { } throw std::runtime_error("could not load corpus"); } +} diff --git a/src/jsonminifier.cpp b/src/jsonminifier.cpp index 20872921b..d4c146464 100644 --- a/src/jsonminifier.cpp +++ b/src/jsonminifier.cpp @@ -1,8 +1,9 @@ #include "simdjson/portability.h" #include + #ifndef __AVX2__ - +namespace simdjson { static uint8_t jump_table[256 * 3] = { 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, @@ -56,12 +57,12 @@ size_t jsonminify(const unsigned char *bytes, size_t howmany, } return pos; } - +} #else - #include "simdjson/simdprune_tables.h" #include +namespace simdjson { // a straightforward comparison of a mask against input. static uint64_t cmp_mask_against_input_mini(__m256i input_lo, __m256i input_hi, __m256i mask) { @@ -247,5 +248,5 @@ size_t jsonminify(const uint8_t *buf, size_t len, uint8_t *out) { *out = '\0';// NULL termination return out - initout; } - +} #endif diff --git a/src/jsonparser.cpp b/src/jsonparser.cpp index c2cf26d98..538ca8130 100644 --- a/src/jsonparser.cpp +++ b/src/jsonparser.cpp @@ -7,44 +7,44 @@ #endif #include "simdjson/simdjson.h" - +namespace simdjson { // Responsible to select the best json_parse implementation int json_parse_dispatch(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded) { // Versions for each implementation #ifdef __AVX2__ - json_parse_functype* avx_implementation = &json_parse_implementation; + json_parse_functype* avx_implementation = &json_parse_implementation; #endif #ifdef __SSE4_2__ - // json_parse_functype* sse4_2_implementation = &json_parse_implementation; // not implemented yet + // json_parse_functype* sse4_2_implementation = &json_parse_implementation; // not implemented yet #endif #ifdef __ARM_NEON - json_parse_functype* neon_implementation = &json_parse_implementation; + json_parse_functype* neon_implementation = &json_parse_implementation; #endif // Determining which implementation is the more suitable // Should be done at runtime. Does not make any sense on preprocessor. #ifdef __AVX2__ - simdjson::instruction_set best_implementation = simdjson::instruction_set::avx2; + instruction_set best_implementation = instruction_set::avx2; #elif defined (__SSE4_2__) - simdjson::instruction_set best_implementation = simdjson::instruction_set::sse4_2; + instruction_set best_implementation = instruction_set::sse4_2; #elif defined (__ARM_NEON) - simdjson::instruction_set best_implementation = simdjson::instruction_set::neon; + instruction_set best_implementation = instruction_set::neon; #else - simdjson::instruction_set best_implementation = simdjson::instruction_set::none; + instruction_set best_implementation = instruction_set::none; #endif // Selecting the best implementation switch (best_implementation) { #ifdef __AVX2__ - case simdjson::instruction_set::avx2 : + case instruction_set::avx2 : json_parse_ptr = avx_implementation; break; #elif defined (__SSE4_2__) - /*case simdjson::instruction_set::sse4_2 : + /*case instruction_set::sse4_2 : json_parse_ptr = sse4_2_implementation; break;*/ #elif defined (__ARM_NEON) - case simdjson::instruction_set::neon : + case instruction_set::neon : json_parse_ptr = neon_implementation; break; #endif @@ -69,3 +69,4 @@ ParsedJson build_parsed_json(const uint8_t *buf, size_t len, bool reallocifneede } return pj; } +} \ No newline at end of file diff --git a/src/parsedjson.cpp b/src/parsedjson.cpp index b0719c78d..d6e661c42 100644 --- a/src/parsedjson.cpp +++ b/src/parsedjson.cpp @@ -1,5 +1,6 @@ #include "simdjson/parsedjson.h" +namespace simdjson { ParsedJson::ParsedJson() : structural_indexes(nullptr), tape(nullptr), containing_scope_offset(nullptr), ret_address(nullptr), string_buf(nullptr), current_string_buf_loc(nullptr) {} @@ -97,7 +98,7 @@ int ParsedJson::getErrorCode() const { } std::string ParsedJson::getErrorMsg() const { - return simdjson::errorMsg(errorcode); + return errorMsg(errorcode); } void ParsedJson::deallocate() { @@ -318,3 +319,4 @@ bool ParsedJson::dump_raw_tape(std::ostream &os) { os << tapeidx << " : "<< type <<"\t// pointing to " << payload <<" (start root)\n"; return true; } +} diff --git a/src/parsedjsoniterator.cpp b/src/parsedjsoniterator.cpp index 72cafcc5c..c92a1ce4e 100644 --- a/src/parsedjsoniterator.cpp +++ b/src/parsedjsoniterator.cpp @@ -2,6 +2,7 @@ #include "simdjson/common_defs.h" #include +namespace simdjson { ParsedJson::iterator::iterator(ParsedJson &pj_) : pj(pj_), depth(0), location(0), tape_length(0), depthindex(nullptr) { if(!pj.isValid()) { throw InvalidJSON(); @@ -92,3 +93,4 @@ bool ParsedJson::iterator::print(std::ostream &os, bool escape_strings) const { } return true; } +} diff --git a/src/simdjson.cpp b/src/simdjson.cpp index 7ba7950eb..d77c3a5b7 100644 --- a/src/simdjson.cpp +++ b/src/simdjson.cpp @@ -1,23 +1,25 @@ #include #include "simdjson/simdjson.h" +namespace simdjson { const std::map errorStrings = { - {simdjson::SUCCESS, "No errors"}, - {simdjson::CAPACITY, "This ParsedJson can't support a document that big"}, - {simdjson::MEMALLOC, "Error allocating memory, we're most likely out of memory"}, - {simdjson::TAPE_ERROR, "Something went wrong while writing to the tape"}, - {simdjson::STRING_ERROR, "Problem while parsing a string"}, - {simdjson::T_ATOM_ERROR, "Problem while parsing an atom starting with the letter 't'"}, - {simdjson::F_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'f'"}, - {simdjson::N_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'n'"}, - {simdjson::NUMBER_ERROR, "Problem while parsing a number"}, - {simdjson::UTF8_ERROR, "The input is not valid UTF-8"}, - {simdjson::UNITIALIZED, "Unitialized"}, - {simdjson::EMPTY, "Empty"}, - {simdjson::UNESCAPED_CHARS, "Within strings, some characters must be escapted, we found unescapted characters"}, - {simdjson::UNEXPECTED_ERROR, "Unexpected error, consider reporting this problem as you may have found a bug in simdjson"}, + {SUCCESS, "No errors"}, + {CAPACITY, "This ParsedJson can't support a document that big"}, + {MEMALLOC, "Error allocating memory, we're most likely out of memory"}, + {TAPE_ERROR, "Something went wrong while writing to the tape"}, + {STRING_ERROR, "Problem while parsing a string"}, + {T_ATOM_ERROR, "Problem while parsing an atom starting with the letter 't'"}, + {F_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'f'"}, + {N_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'n'"}, + {NUMBER_ERROR, "Problem while parsing a number"}, + {UTF8_ERROR, "The input is not valid UTF-8"}, + {UNITIALIZED, "Unitialized"}, + {EMPTY, "Empty"}, + {UNESCAPED_CHARS, "Within strings, some characters must be escapted, we found unescapted characters"}, + {UNEXPECTED_ERROR, "Unexpected error, consider reporting this problem as you may have found a bug in simdjson"}, }; -const std::string& simdjson::errorMsg(const int errorCode) { +const std::string& errorMsg(const int errorCode) { return errorStrings.at(errorCode); -} \ No newline at end of file +} +} diff --git a/src/stage2_build_tape.cpp b/src/stage2_build_tape.cpp index 550449a23..9b844cdda 100644 --- a/src/stage2_build_tape.cpp +++ b/src/stage2_build_tape.cpp @@ -11,6 +11,7 @@ #include #define PATH_SEP '/' +namespace simdjson { WARN_UNUSED really_inline bool is_valid_true_atom(const uint8_t *loc) { @@ -82,7 +83,7 @@ int unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) { uint32_t depth = 0; // could have an arbitrary starting depth pj.init(); // sets isvalid to false if(pj.bytecapacity < len) { - pj.errorcode = simdjson::CAPACITY; + pj.errorcode = CAPACITY; return pj.errorcode; } // this macro reads the next structural character, updating idx, i and c. @@ -526,7 +527,7 @@ succeed: pj.write_tape(pj.containing_scope_offset[depth], 'r'); // r is root pj.isvalid = true; - pj.errorcode = simdjson::SUCCESS; + pj.errorcode = SUCCESS; return pj.errorcode; fail: // we do not need the next line because this is done by pj.init(), pessimistically. @@ -537,12 +538,12 @@ fail: // We could even trigger special code paths to assess what happened carefully, // all without any added cost. if (depth >= pj.depthcapacity) { - pj.errorcode = simdjson::DEPTH_ERROR; + pj.errorcode = DEPTH_ERROR; return pj.errorcode; } switch(c) { case '"': - pj.errorcode = simdjson::STRING_ERROR; + pj.errorcode = STRING_ERROR; return pj.errorcode; case '0': case '1': @@ -555,24 +556,25 @@ fail: case '8': case '9': case '-': - pj.errorcode = simdjson::NUMBER_ERROR; + pj.errorcode = NUMBER_ERROR; return pj.errorcode; case 't': - pj.errorcode = simdjson::T_ATOM_ERROR; + pj.errorcode = T_ATOM_ERROR; return pj.errorcode; case 'n': - pj.errorcode = simdjson::N_ATOM_ERROR; + pj.errorcode = N_ATOM_ERROR; return pj.errorcode; case 'f': - pj.errorcode = simdjson::F_ATOM_ERROR; + pj.errorcode = F_ATOM_ERROR; return pj.errorcode; default: break; } - pj.errorcode = simdjson::TAPE_ERROR; + pj.errorcode = TAPE_ERROR; return pj.errorcode; } int unified_machine(const char *buf, size_t len, ParsedJson &pj) { return unified_machine(reinterpret_cast(buf), len, pj); } +} diff --git a/tests/allparserscheckfile.cpp b/tests/allparserscheckfile.cpp index 1e7d83ca9..d67cb96f4 100644 --- a/tests/allparserscheckfile.cpp +++ b/tests/allparserscheckfile.cpp @@ -59,9 +59,9 @@ int main(int argc, char *argv[]) { exit(1); } const char *filename = argv[optind]; - padded_string p; + simdjson::padded_string p; try { - get_corpus(filename).swap(p); + simdjson::get_corpus(filename).swap(p); } catch (const std::exception &e) { // caught by reference to base std::cout << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; @@ -76,7 +76,7 @@ int main(int argc, char *argv[]) { std::cout << p.size() << " B "; std::cout << std::endl; } - ParsedJson pj; + simdjson::ParsedJson pj; size_t maxdepth = 1024 * 4; bool allocok = pj.allocateCapacity(p.size(), maxdepth); if (!allocok) { diff --git a/tests/basictests.cpp b/tests/basictests.cpp index fd2eff5e0..10bd07320 100644 --- a/tests/basictests.cpp +++ b/tests/basictests.cpp @@ -39,7 +39,7 @@ bool skyprophet_test() { if (maxsize < s.size()) maxsize = s.size(); } - ParsedJson pj; + simdjson::ParsedJson pj; if (!pj.allocateCapacity(maxsize)) { printf("allocation failure in skyprophet_test\n"); return false; diff --git a/tests/jsoncheck.cpp b/tests/jsoncheck.cpp index d3d1dfaa2..8646a5fcb 100644 --- a/tests/jsoncheck.cpp +++ b/tests/jsoncheck.cpp @@ -65,14 +65,14 @@ bool validate(const char *dirname) { } else { strcpy(fullpath + dirlen, name); } - padded_string p; + simdjson::padded_string p; try { - get_corpus(fullpath).swap(p); + simdjson::get_corpus(fullpath).swap(p); } catch (const std::exception &e) { std::cerr << "Could not load the file " << fullpath << std::endl; return EXIT_FAILURE; } - ParsedJson pj; + simdjson::ParsedJson pj; bool allocok = pj.allocateCapacity(p.size(), 1024); if (!allocok) { std::cerr << "can't allocate memory" << std::endl; diff --git a/tests/numberparsingcheck.cpp b/tests/numberparsingcheck.cpp index 3fcc22f0a..8cbfee88d 100644 --- a/tests/numberparsingcheck.cpp +++ b/tests/numberparsingcheck.cpp @@ -132,15 +132,15 @@ bool validate(const char *dirname) { } else { strcpy(fullpath + dirlen, name); } - padded_string p; + simdjson::padded_string p; try { - get_corpus(fullpath).swap(p); + simdjson::get_corpus(fullpath).swap(p); } catch (const std::exception &e) { std::cout << "Could not load the file " << fullpath << std::endl; return EXIT_FAILURE; } // terrible hack but just to get it working - ParsedJson pj; + simdjson::ParsedJson pj; bool allocok = pj.allocateCapacity(p.size(), 1024); if (!allocok) { std::cerr << "can't allocate memory" << std::endl; diff --git a/tests/stringparsingcheck.cpp b/tests/stringparsingcheck.cpp index 88e3521d3..5beec83bd 100644 --- a/tests/stringparsingcheck.cpp +++ b/tests/stringparsingcheck.cpp @@ -325,14 +325,14 @@ bool validate(const char *dirname) { } else { strcpy(fullpath + dirlen, name); } - padded_string p; + simdjson::padded_string p; try { - get_corpus(fullpath).swap(p); + simdjson::get_corpus(fullpath).swap(p); } catch (const std::exception &e) { std::cout << "Could not load the file " << fullpath << std::endl; return EXIT_FAILURE; } - ParsedJson pj; + simdjson::ParsedJson pj; bool allocok = pj.allocateCapacity(p.size(), 1024); if (!allocok) { std::cerr << "can't allocate memory" << std::endl; diff --git a/tools/json2json.cpp b/tools/json2json.cpp index a01a6d8ea..ebe4183bc 100644 --- a/tools/json2json.cpp +++ b/tools/json2json.cpp @@ -5,7 +5,7 @@ #include "simdjson/jsonioutil.h" #include "simdjson/jsonparser.h" -void compute_dump(ParsedJson::iterator &pjh) { +void compute_dump(simdjson::ParsedJson::iterator &pjh) { if (pjh.is_object()) { std::cout << "{"; if (pjh.down()) { @@ -72,26 +72,26 @@ int main(int argc, char *argv[]) { if (optind + 1 < argc) { std::cerr << "warning: ignoring everything after " << argv[optind + 1] << std::endl; } - padded_string p; + simdjson::padded_string p; try { - get_corpus(filename).swap(p); + simdjson::get_corpus(filename).swap(p); } catch (const std::exception &e) { // caught by reference to base std::cout << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; } - ParsedJson pj; + simdjson::ParsedJson pj; bool allocok = pj.allocateCapacity(p.size(), 1024); if (!allocok) { std::cerr << "failed to allocate memory" << std::endl; return EXIT_FAILURE; } - int res = json_parse(p, pj); // do the parsing, return false on error + int res = simdjson::json_parse(p, pj); // do the parsing, return false on error if (res) { std::cerr << " Parsing failed. " << std::endl; return EXIT_FAILURE; } if (apidump) { - ParsedJson::iterator pjh(pj); + simdjson::ParsedJson::iterator pjh(pj); if (!pjh.isOk()) { std::cerr << " Could not iterate parsed result. " << std::endl; return EXIT_FAILURE; diff --git a/tools/jsonstats.cpp b/tools/jsonstats.cpp index 8e7462097..ed826891f 100644 --- a/tools/jsonstats.cpp +++ b/tools/jsonstats.cpp @@ -41,9 +41,9 @@ using stat_t = struct stat_s; -stat_t simdjson_computestats(const padded_string &p) { +stat_t simdjson_computestats(const simdjson::padded_string &p) { stat_t answer; - ParsedJson pj = build_parsed_json(p); + simdjson::ParsedJson pj = simdjson::build_parsed_json(p); answer.valid = pj.isValid(); if (!answer.valid) { return answer; @@ -125,9 +125,9 @@ int main(int argc, char *argv[]) { if (myoptind + 1 < argc) { std::cerr << "warning: ignoring everything after " << argv[myoptind + 1] << std::endl; } - padded_string p; + simdjson::padded_string p; try { - get_corpus(filename).swap(p); + simdjson::get_corpus(filename).swap(p); } catch (const std::exception &e) { // caught by reference to base std::cerr << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; diff --git a/tools/minify.cpp b/tools/minify.cpp index 56263b788..b5940482e 100644 --- a/tools/minify.cpp +++ b/tools/minify.cpp @@ -8,14 +8,14 @@ int main(int argc, char *argv[]) { std::cerr << "Usage: " << argv[0] << " \n"; exit(1); } - padded_string p; + simdjson::padded_string p; std::string filename = argv[argc - 1]; try{ - get_corpus(filename).swap(p); + simdjson::get_corpus(filename).swap(p); } catch (const std::exception& e) { std::cout << "Could not load the file " << filename << std::endl; return EXIT_FAILURE; } - jsonminify(p, p.data()); + simdjson::jsonminify(p, p.data()); printf("%s",p.data()); }