From 52c4b65f1eeaab2bf146eafb768fa7033a52182d Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Sun, 9 Dec 2018 20:47:02 -0500 Subject: [PATCH] Progress validating the API. --- include/simdjson/jsonformatutils.h | 40 +++++++++++++++++++ include/simdjson/parsedjson.h | 38 ++++++++++++------ scripts/testjson2json.sh | 63 +++++++++++++++++++++++++++++- tools/json2json.cpp | 11 +++++- 4 files changed, 137 insertions(+), 15 deletions(-) diff --git a/include/simdjson/jsonformatutils.h b/include/simdjson/jsonformatutils.h index be7aedd86..fa65fa33b 100644 --- a/include/simdjson/jsonformatutils.h +++ b/include/simdjson/jsonformatutils.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include static inline void print_with_escapes(const unsigned char *src) { while (*src) { @@ -35,3 +37,41 @@ static inline void print_with_escapes(const unsigned char *src) { } } +static inline void print_with_escapes(const unsigned char *src, std::ostream &os) { + while (*src) { + switch (*src) { + case '\n': + os << '\\'; + os << 'n'; + break; + case '\r': + os << '\\'; + os << 'r'; + break; + case '\"': + os << '\\'; + os << '"'; + break; + case '\t': + os << '\\'; + os << 't'; + break; + case '\\': + os << '\\'; + os << '\\'; + break; + default: + if (*src <= 0x1F) { + std::ios::fmtflags f(os.flags()); + os << std::hex << std::setw(4) << std::setfill('0') << (int) *src; + os.flags(f); + } else + os << *src; + } + src++; + } +} + +static inline void print_with_escapes(const char *src, std::ostream &os) { + print_with_escapes((const unsigned char *)src, os); +} \ No newline at end of file diff --git a/include/simdjson/parsedjson.h b/include/simdjson/parsedjson.h index ee69578d7..a80e8f3d7 100644 --- a/include/simdjson/parsedjson.h +++ b/include/simdjson/parsedjson.h @@ -264,16 +264,16 @@ public: printf("false\n"); break; case '{': // we have an object - printf("{\n"); + printf("{\t// pointing to next tape location %llu \n", payload); break; case '}': // we end an object - printf("}\n"); + printf("}\t// pointing to previous tape location %llu \n", payload); break; case '[': // we start an array - printf("[\n"); + printf("[\t// pointing to next tape location %llu \n", payload); break; case ']': // we end an array - printf("]\n"); + printf("]\t// pointing to previous tape location %llu \n", payload); break; case 'r': // we start and end with the root node printf("end of root\n"); @@ -368,7 +368,7 @@ public: current_val(o.current_val), depthindex(o.depthindex) { o.depthindex = NULL;// we take ownship } - + WARN_UNUSED bool isOk() const { return location < tape_length; } @@ -384,7 +384,9 @@ public: // return true if we can do the navigation, false // otherwise - // valid if we're not at the end of a scope + // withing a give scope, we move forward + // valid if we're not at the end of a scope (returns true) + WARN_UNUSED really_inline bool next() { if ((current_type == '[') || (current_type == '{')){ // we need to jump @@ -417,6 +419,7 @@ public: } // valid if we're not at the start of a scope + WARN_UNUSED really_inline bool prev() { if(location - 1 < depthindex[depth]) return false; location -= 1; @@ -437,6 +440,7 @@ public: // valid unless we are at the first level of the document + WARN_UNUSED really_inline bool up() { if(depth == 1) { return false; // don't allow moving back to root @@ -451,11 +455,16 @@ public: } - // valid if we're at a [ or { call site; moves us to start of - // that scope + // valid if we're at a [ or { and it starts a non-empty scope; moves us to start of + // that deeper scope if it not empty + WARN_UNUSED really_inline bool down() { if(location + 1 >= tape_length) return false; if ((current_type == '[') || (current_type == '{')) { + size_t npos = (current_val & JSONVALUEMASK); + if(npos == location + 2) { + return false; // we have an empty scope + } depth++; location = location + 1; depthindex[depth] = location; @@ -477,17 +486,24 @@ public: // the start of our current scope; always succeeds // print the thing we're currently pointing at - bool print(std::ostream &os) const { + bool print(std::ostream &os, bool escape_strings = true) const { if(!isOk()) return false; switch (current_type) { case '"': // we have a string - os << '"' << get_string() << '"'; + os << '"'; + if(escape_strings) { + print_with_escapes(get_string(), os); + } else { + os << get_string(); + } + os << '"'; break; case 'l': // we have a long int os << get_integer(); break; case 'd': - os << get_double(); + os << get_double(); + break; case 'n': // we have a null os << "null"; break; diff --git a/scripts/testjson2json.sh b/scripts/testjson2json.sh index f7c5299f7..0e35c3fa9 100755 --- a/scripts/testjson2json.sh +++ b/scripts/testjson2json.sh @@ -2,9 +2,10 @@ TMPDIR1=$(mktemp -d -t simdjsonXXXXXXXX) TMPDIR2=$(mktemp -d -t simdjsonXXXXXXXX) - +TMPDIR3=$(mktemp -d -t simdjsonXXXXXXXX) +TMPDIR4=$(mktemp -d -t simdjsonXXXXXXXX) trap "exit 1" HUP INT PIPE QUIT TERM -trap "rm -rf $TMPDIR1 $TMPDIR2" EXIT +trap "rm -rf $TMPDIR1 $TMPDIR2 $TMPDIR3 $TMPDIR4" EXIT function founderror() { echo "code is wrong" @@ -15,19 +16,33 @@ make minify json2json for i in `cd jsonexamples && ls -1 *.json`; do echo $i ./json2json jsonexamples/$i > $TMPDIR1/$i + ./json2json -a jsonexamples/$i > $TMPDIR3/$i ./json2json $TMPDIR1/$i > $TMPDIR2/$i + ./json2json -a $TMPDIR3/$i > $TMPDIR4/$i cmp $TMPDIR1/$i $TMPDIR2/$i retVal=$? if [ $retVal -ne 0 ]; then founderror fi + cmp $TMPDIR3/$i $TMPDIR4/$i + retVal=$? + if [ $retVal -ne 0 ]; then + founderror + fi ./minify $TMPDIR1/$i > $TMPDIR1/minify$i ./minify $TMPDIR2/$i > $TMPDIR2/minify$i + ./minify $TMPDIR3/$i > $TMPDIR3/minify$i + ./minify $TMPDIR4/$i > $TMPDIR4/minify$i cmp $TMPDIR1/minify$i $TMPDIR2/minify$i retVal=$? if [ $retVal -ne 0 ]; then founderror fi + cmp $TMPDIR3/minify$i $TMPDIR4/minify$i + retVal=$? + if [ $retVal -ne 0 ]; then + founderror + fi ./json2json $TMPDIR1/minify$i > $TMPDIR2/bisminify$i cmp $TMPDIR1/$i $TMPDIR2/bisminify$i retVal=$? @@ -35,6 +50,50 @@ for i in `cd jsonexamples && ls -1 *.json`; do founderror fi done + +for i in `cd jsonchecker && ls -1 pass*.json`; do + echo $i + ./json2json jsonchecker/$i > $TMPDIR1/$i + ./json2json -a jsonchecker/$i > $TMPDIR3/$i + ./json2json $TMPDIR1/$i > $TMPDIR2/$i + ./json2json -a $TMPDIR3/$i > $TMPDIR4/$i + cmp $TMPDIR1/$i $TMPDIR2/$i + retVal=$? + if [ $retVal -ne 0 ]; then + echo "reg failure" + founderror + fi + cmp $TMPDIR3/$i $TMPDIR4/$i + retVal=$? + if [ $retVal -ne 0 ]; then + echo "-a failure" + founderror + fi + ./minify $TMPDIR1/$i > $TMPDIR1/minify$i + ./minify $TMPDIR2/$i > $TMPDIR2/minify$i + ./minify $TMPDIR3/$i > $TMPDIR3/minify$i + ./minify $TMPDIR4/$i > $TMPDIR4/minify$i + cmp $TMPDIR1/minify$i $TMPDIR2/minify$i + retVal=$? + if [ $retVal -ne 0 ]; then + echo "reg failure, step 2" + founderror + fi + cmp $TMPDIR3/minify$i $TMPDIR4/minify$i + retVal=$? + if [ $retVal -ne 0 ]; then + echo "-a failure, step 2" + founderror + fi + ./json2json $TMPDIR1/minify$i > $TMPDIR2/bisminify$i + cmp $TMPDIR1/$i $TMPDIR2/bisminify$i + retVal=$? + if [ $retVal -ne 0 ]; then + founderror + fi +done + + echo "test successful" exit 0 diff --git a/tools/json2json.cpp b/tools/json2json.cpp index 853d887d6..de2977ebf 100644 --- a/tools/json2json.cpp +++ b/tools/json2json.cpp @@ -14,7 +14,14 @@ void compute_dump(ParsedJson::iterator &pjh) { return; // we are done } // we have either an array or an object - pjh.down(); + bool goingdown = pjh.down(); + if(!goingdown) { + // we have an empty scope + if(inobject) std::cout<<"{}"; + else std::cout<<"[]"; + return; + } + // we have a non-empty scope and we are at the beginning of it if (inobject) { std::cout << "{"; assert(pjh.get_type() == '"'); @@ -40,7 +47,7 @@ void compute_dump(ParsedJson::iterator &pjh) { } std::cout << "]"; } - pjh.up(); + assert(pjh.up()); } int main(int argc, char *argv[]) {