diff --git a/benchmark/distinctuseridcompetition.cpp b/benchmark/distinctuseridcompetition.cpp index 9dd187897..a99f11d12 100644 --- a/benchmark/distinctuseridcompetition.cpp +++ b/benchmark/distinctuseridcompetition.cpp @@ -36,15 +36,15 @@ void simdjson_traverse(std::vector &answer, ParsedJson::iterator &i) { if (i.down()) { do { bool founduser = (i.get_string_length() == 4) && (memcmp(i.get_string(), "user", 4) == 0); - i.next(); // move to value + i.move_to_value(); // move to value if (i.is_object()) { if (founduser && i.move_to_key("id")) { if (i.is_integer()) { answer.push_back(i.get_integer()); } i.up(); - } - simdjson_traverse(answer, i); + } + simdjson_traverse(answer, i); } else if (i.is_array()) { simdjson_traverse(answer, i); } @@ -72,6 +72,17 @@ void simdjson_traverse(std::vector &answer, ParsedJson::iterator &i) { } } +__attribute__ ((noinline)) +std::vector simdjson_justdom(ParsedJson &pj) { + std::vector answer; + ParsedJson::iterator i(pj); + + simdjson_traverse(answer, i); + remove_duplicates(answer); + return answer; +} + +__attribute__ ((noinline)) std::vector simdjson_computestats(const padded_string &p) { std::vector answer; ParsedJson pj = build_parsed_json(p); @@ -85,6 +96,13 @@ std::vector simdjson_computestats(const padded_string &p) { return answer; } +__attribute__ ((noinline)) +bool simdjson_justparse(const padded_string &p) { + ParsedJson pj = build_parsed_json(p); + bool answer = !pj.isValid(); + return answer; +} + void sajson_traverse(std::vector &answer, const sajson::value &node) { using namespace sajson; switch (node.get_type()) { @@ -97,7 +115,8 @@ void sajson_traverse(std::vector &answer, const sajson::value &node) { } case TYPE_OBJECT: { auto length = node.get_length(); - // sajson has O(log n) find_object_key, but we still visit each node anyhow + // sajson has O(log n) find_object_key, but we still visit each node anyhow because we + // need to visit all values. for (auto i = 0u; i < length; ++i) { auto key = node.get_object_key(i); // expected: sajson::string bool founduser = (key.length() == 4) && (memcmp(key.data(), "user", 4) == 0); @@ -105,18 +124,16 @@ void sajson_traverse(std::vector &answer, const sajson::value &node) { auto uservalue = node.get_object_value(i); // get the value if (uservalue.get_type() == TYPE_OBJECT) { // the value should be an object + // now we know that we only need one value auto uservaluelength = uservalue.get_length(); - for (auto j = 0u; j < uservaluelength; - ++j) { // go through the children - if (equals(uservalue.get_object_key(j).data(), - "id")) { // ah ah found id - auto v = uservalue.get_object_value(j); + auto rightindex = uservalue.find_object_key(sajson::string("id",2)); + if(rightindex < uservaluelength) { + auto v = uservalue.get_object_value(rightindex); if (v.get_type() == TYPE_INTEGER) { // check that it is an integer answer.push_back(v.get_integer_value()); // record it! } else if (v.get_type() == TYPE_DOUBLE) { answer.push_back((int64_t)v.get_double_value()); // record it! } - } } } } @@ -136,6 +153,15 @@ void sajson_traverse(std::vector &answer, const sajson::value &node) { } } +__attribute__ ((noinline)) +std::vector sasjon_justdom(sajson::document & d) { + std::vector answer; + sajson_traverse(answer, d.get_root()); + remove_duplicates(answer); + return answer; +} + +__attribute__ ((noinline)) std::vector sasjon_computestats(const padded_string &p) { std::vector answer; char *buffer = (char *)malloc(p.size()); @@ -143,6 +169,7 @@ std::vector sasjon_computestats(const padded_string &p) { auto d = sajson::parse(sajson::dynamic_allocation(), sajson::mutable_string_view(p.size(), buffer)); if (!d.is_valid()) { + free(buffer); return answer; } sajson_traverse(answer, d.get_root()); @@ -151,6 +178,17 @@ std::vector sasjon_computestats(const padded_string &p) { return answer; } +__attribute__ ((noinline)) +bool sasjon_justparse(const padded_string &p) { + char *buffer = (char *)malloc(p.size()); + memcpy(buffer, p.data(), p.size()); + auto d = sajson::parse(sajson::dynamic_allocation(), + sajson::mutable_string_view(p.size(), buffer)); + bool answer = !d.is_valid(); + free(buffer); + return answer; +} + void rapid_traverse(std::vector &answer, const rapidjson::Value &v) { switch (v.GetType()) { case kObjectType: @@ -190,6 +228,15 @@ void rapid_traverse(std::vector &answer, const rapidjson::Value &v) { } } +__attribute__ ((noinline)) +std::vector rapid_justdom(rapidjson::Document &d) { + std::vector answer; + rapid_traverse(answer, d); + remove_duplicates(answer); + return answer; +} + +__attribute__ ((noinline)) std::vector rapid_computestats(const padded_string &p) { std::vector answer; char *buffer = (char *)malloc(p.size() + 1); @@ -198,7 +245,8 @@ std::vector rapid_computestats(const padded_string &p) { rapidjson::Document d; d.ParseInsitu(buffer); if (d.HasParseError()) { - return answer; + free(buffer); + return answer; } rapid_traverse(answer, d); free(buffer); @@ -206,6 +254,19 @@ std::vector rapid_computestats(const padded_string &p) { return answer; } +__attribute__ ((noinline)) +bool rapid_justparse(const padded_string &p) { + char *buffer = (char *)malloc(p.size() + 1); + memcpy(buffer, p.data(), p.size()); + buffer[p.size()] = '\0'; + rapidjson::Document d; + d.ParseInsitu(buffer); + bool answer = d.HasParseError(); + free(buffer); + return answer; +} + + int main(int argc, char *argv[]) { bool verbose = false; bool justdata = false; @@ -270,7 +331,7 @@ int main(int argc, char *argv[]) { assert(s1 == s3); size_t size = s1.size(); - int repeat = 50; + int repeat = 500; int volume = p.size(); if(justdata) { printf("name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err \n"); @@ -282,4 +343,25 @@ int main(int argc, char *argv[]) { !justdata); BEST_TIME("sasjon ", sasjon_computestats(p).size(), size, , repeat, volume, !justdata); + BEST_TIME("simdjson (just parse) ", simdjson_justparse(p), false, , repeat, + volume, !justdata); + BEST_TIME("rapid (just parse) ", rapid_justparse(p), false, , repeat, volume, + !justdata); + BEST_TIME("sasjon (just parse) ", sasjon_justparse(p), false, , repeat, volume, + !justdata); + ParsedJson dsimdjson = build_parsed_json(p); + BEST_TIME("simdjson (just dom) ", simdjson_justdom(dsimdjson).size(), size, , repeat, + volume, !justdata); + char *buffer = (char *)malloc(p.size()); + memcpy(buffer, p.data(), p.size()); + rapidjson::Document drapid; + drapid.ParseInsitu(buffer); + BEST_TIME("rapid (just dom) ", rapid_justdom(drapid).size(), size, , repeat, volume, + !justdata); + memcpy(buffer, p.data(), p.size()); + auto dsasjon = sajson::parse(sajson::dynamic_allocation(), + sajson::mutable_string_view(p.size(), buffer)); + BEST_TIME("sasjon (just dom) ", sasjon_justdom(dsasjon).size(), size, , repeat, volume, + !justdata); + free(buffer); } diff --git a/include/simdjson/parsedjson.h b/include/simdjson/parsedjson.h index 40ded9b51..e74be117c 100644 --- a/include/simdjson/parsedjson.h +++ b/include/simdjson/parsedjson.h @@ -104,23 +104,23 @@ public: iterator(iterator &&o); - bool isOk() const; + inline bool isOk() const; // useful for debuging purposes - size_t get_tape_location() const; + inline size_t get_tape_location() const; // useful for debuging purposes - size_t get_tape_length() const; + inline size_t get_tape_length() const; // returns the current depth (start at 1 with 0 reserved for the fictitious root node) - size_t get_depth() const; + inline size_t get_depth() const; // A scope is a series of nodes at the same depth, typically it is either an object ({) or an array ([). // The root node has type 'r'. - uint8_t get_scope_type() const; + inline uint8_t get_scope_type() const; // move forward in document order - bool move_forward(); + inline bool move_forward(); // retrieve the character code of what we're looking at: // [{"sltfn are the possibilities @@ -211,7 +211,11 @@ public: // We seek the key using C's strcmp so if your JSON strings contain // NULL chars, this would trigger a false positive: if you expect that // to be the case, take extra precautions. - bool move_to_key(const char * key); + inline bool move_to_key(const char * key); + + // when at a key location within an object, this moves to the accompanying value (located next to it). + // this is equivalent but much faster than calling "next()". + inline void move_to_value(); // throughout return true if we can do the navigation, false // otherwise @@ -221,30 +225,30 @@ public: // Thus, given [true, null, {"a":1}, [1,2]], we would visit true, null, { and [. // At the object ({) or at the array ([), you can issue a "down" to visit their content. // valid if we're not at the end of a scope (returns true). - bool next(); + inline bool next(); // Withing a given scope (series of nodes at the same depth within either an // array or an object), we move backward. // Thus, given [true, null, {"a":1}, [1,2]], we would visit ], }, null, true when starting at the end // of the scope. // At the object ({) or at the array ([), you can issue a "down" to visit their content. - bool prev(); + inline bool prev(); // Moves back to either the containing array or object (type { or [) from // within a contained scope. // Valid unless we are at the first level of the document - bool up(); + inline bool up(); // Valid if we're at a [ or { and it starts a non-empty scope; moves us to start of // that deeper scope if it not empty. // Thus, given [true, null, {"a":1}, [1,2]], if we are at the { node, we would move to the // "a" node. - bool down(); + inline bool down(); // move us to the start of our current scope, // a scope is a series of nodes at the same level - void to_start_scope(); + inline void to_start_scope(); // void to_end_scope(); // move us to // the start of our current scope; always succeeds @@ -312,5 +316,161 @@ inline void dumpbits32_always(uint32_t v, const std::string &msg) { std::cout << " " << msg.c_str() << "\n"; } +WARN_UNUSED +bool ParsedJson::iterator::isOk() const { + return location < tape_length; +} +// useful for debuging purposes +size_t ParsedJson::iterator::get_tape_location() const { + return location; +} + +// useful for debuging purposes +size_t ParsedJson::iterator::get_tape_length() const { + return tape_length; +} + +// returns the current depth (start at 1 with 0 reserved for the fictitious root node) +size_t ParsedJson::iterator::get_depth() const { + return depth; +} + +// A scope is a series of nodes at the same depth, typically it is either an object ({) or an array ([). +// The root node has type 'r'. +uint8_t ParsedJson::iterator::get_scope_type() const { + return depthindex[depth].scope_type; +} + +bool ParsedJson::iterator::move_forward() { + if(location + 1 >= tape_length) { + return false; // we are at the end! + } + + if ((current_type == '[') || (current_type == '{')){ + // We are entering a new scope + depth++; + depthindex[depth].start_of_scope = location; + depthindex[depth].scope_type = current_type; + } else if ((current_type == ']') || (current_type == '}')) { + // Leaving a scope. + depth--; + if(depth == 0) { + // Should not be necessary + return false; + } + } else if ((current_type == 'd') || (current_type == 'l')) { + // d and l types use 2 locations on the tape, not just one. + location += 1; + } + + location += 1; + current_val = pj.tape[location]; + current_type = (current_val >> 56); + return true; +} + +void ParsedJson::iterator::move_to_value() { + // assume that we are on a key, so move by 1. + location += 1; + current_val = pj.tape[location]; + current_type = (current_val >> 56); +} + + +bool ParsedJson::iterator::move_to_key(const char * key) { + if(down()) { + do { + assert(is_string()); + bool rightkey = (strcmp(get_string(),key)==0);// null chars would fool this + move_to_value(); + if(rightkey) { + return true; + } + } while(next()); + assert(up());// not found + } + return false; +} + + + bool ParsedJson::iterator::prev() { + if(location - 1 < depthindex[depth].start_of_scope) { + return false; + } + location -= 1; + current_val = pj.tape[location]; + current_type = (current_val >> 56); + if ((current_type == ']') || (current_type == '}')){ + // we need to jump + size_t new_location = ( current_val & JSONVALUEMASK); + if(new_location < depthindex[depth].start_of_scope) { + return false; // shoud never happen + } + location = new_location; + current_val = pj.tape[location]; + current_type = (current_val >> 56); + } + return true; +} + + + bool ParsedJson::iterator::up() { + if(depth == 1) { + return false; // don't allow moving back to root + } + to_start_scope(); + // next we just move to the previous value + depth--; + location -= 1; + current_val = pj.tape[location]; + current_type = (current_val >> 56); + return true; +} + + + bool ParsedJson::iterator::down() { + if(location + 1 >= tape_length) { + return false; + } + if ((current_type == '[') || (current_type == '{')) { + size_t npos = (current_val & JSONVALUEMASK); + if(npos == location + 2) { + return false; // we have an empty scope + } + depth++; + location = location + 1; + depthindex[depth].start_of_scope = location; + depthindex[depth].scope_type = current_type; + current_val = pj.tape[location]; + current_type = (current_val >> 56); + return true; + } + return false; +} + +void ParsedJson::iterator::to_start_scope() { + location = depthindex[depth].start_of_scope; + current_val = pj.tape[location]; + current_type = (current_val >> 56); +} + +bool ParsedJson::iterator::next() { + size_t npos; // next position + if ((current_type == '[') || (current_type == '{')){ + // we need to jump + npos = ( current_val & JSONVALUEMASK); + } else { + npos = location + ((current_type == 'd' || current_type == 'l') ? 2 : 1); + } + uint64_t nextval = pj.tape[npos]; + uint8_t nexttype = (nextval >> 56); + if((nexttype == ']') || (nexttype == '}')) { + return false; // we reached the end of the scope + } + location = npos; + current_val = nextval; + current_type = nexttype; + return true; +} #endif diff --git a/src/parsedjsoniterator.cpp b/src/parsedjsoniterator.cpp index 60286b748..72cafcc5c 100644 --- a/src/parsedjsoniterator.cpp +++ b/src/parsedjsoniterator.cpp @@ -51,170 +51,6 @@ ParsedJson::iterator::iterator(iterator &&o): o.depthindex = nullptr;// we take ownership } -WARN_UNUSED -bool ParsedJson::iterator::isOk() const { - return location < tape_length; -} - -// useful for debuging purposes -size_t ParsedJson::iterator::get_tape_location() const { - return location; -} - -// useful for debuging purposes -size_t ParsedJson::iterator::get_tape_length() const { - return tape_length; -} - -// returns the current depth (start at 1 with 0 reserved for the fictitious root node) -size_t ParsedJson::iterator::get_depth() const { - return depth; -} - -// A scope is a series of nodes at the same depth, typically it is either an object ({) or an array ([). -// The root node has type 'r'. -uint8_t ParsedJson::iterator::get_scope_type() const { - return depthindex[depth].scope_type; -} - -bool ParsedJson::iterator::move_forward() { - if(location + 1 >= tape_length) { - return false; // we are at the end! - } - - if ((current_type == '[') || (current_type == '{')){ - // We are entering a new scope - depth++; - depthindex[depth].start_of_scope = location; - depthindex[depth].scope_type = current_type; - } else if ((current_type == ']') || (current_type == '}')) { - // Leaving a scope. - depth--; - if(depth == 0) { - // Should not be necessary - return false; - } - } else if ((current_type == 'd') || (current_type == 'l')) { - // d and l types use 2 locations on the tape, not just one. - location += 1; - } - - location += 1; - current_val = pj.tape[location]; - current_type = (current_val >> 56); - return true; -} - -bool ParsedJson::iterator::move_to_key(const char * key) { - if(down()) { - do { - assert(is_string()); - bool rightkey = (strcmp(get_string(),key)==0);// null chars would fool this - next(); - if(rightkey) { - return true; - } - } while(next()); - assert(up());// not found - } - return false; -} - - - bool ParsedJson::iterator::next() { - if ((current_type == '[') || (current_type == '{')){ - // we need to jump - size_t npos = ( current_val & JSONVALUEMASK); - if(npos >= tape_length) { - return false; // shoud never happen unless at the root - } - uint64_t nextval = pj.tape[npos]; - uint8_t nexttype = (nextval >> 56); - if((nexttype == ']') || (nexttype == '}')) { - return false; // we reached the end of the scope - } - location = npos; - current_val = nextval; - current_type = nexttype; - return true; - } - size_t increment = (current_type == 'd' || current_type == 'l') ? 2 : 1; - if(location + increment >= tape_length) { - return false; - } - uint64_t nextval = pj.tape[location + increment]; - uint8_t nexttype = (nextval >> 56); - if((nexttype == ']') || (nexttype == '}')) { - return false; // we reached the end of the scope - } - location = location + increment; - current_val = nextval; - current_type = nexttype; - return true; -} - - - bool ParsedJson::iterator::prev() { - if(location - 1 < depthindex[depth].start_of_scope) { - return false; - } - location -= 1; - current_val = pj.tape[location]; - current_type = (current_val >> 56); - if ((current_type == ']') || (current_type == '}')){ - // we need to jump - size_t new_location = ( current_val & JSONVALUEMASK); - if(new_location < depthindex[depth].start_of_scope) { - return false; // shoud never happen - } - location = new_location; - current_val = pj.tape[location]; - current_type = (current_val >> 56); - } - return true; -} - - - bool ParsedJson::iterator::up() { - if(depth == 1) { - return false; // don't allow moving back to root - } - to_start_scope(); - // next we just move to the previous value - depth--; - location -= 1; - current_val = pj.tape[location]; - current_type = (current_val >> 56); - return true; -} - - - bool ParsedJson::iterator::down() { - if(location + 1 >= tape_length) { - return false; - } - if ((current_type == '[') || (current_type == '{')) { - size_t npos = (current_val & JSONVALUEMASK); - if(npos == location + 2) { - return false; // we have an empty scope - } - depth++; - location = location + 1; - depthindex[depth].start_of_scope = location; - depthindex[depth].scope_type = current_type; - current_val = pj.tape[location]; - current_type = (current_val >> 56); - return true; - } - return false; -} - -void ParsedJson::iterator::to_start_scope() { - location = depthindex[depth].start_of_scope; - current_val = pj.tape[location]; - current_type = (current_val >> 56); -} - bool ParsedJson::iterator::print(std::ostream &os, bool escape_strings) const { if(!isOk()) { return false;