admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Progress validating the API.

Browse Source
This commit is contained in:
Daniel Lemire
2018-12-09 20:47:02 -05:00
parent a56e92a571
commit 52c4b65f1e
4 changed files with 137 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/simdjson/jsonformatutils.h
+40
View File
@@ -1,6 +1,8 @@
#pragma once
#include <stdio.h>
#include <iostream>
#include <iomanip>
static inline void print_with_escapes(const unsigned char *src) {
while (*src) {
@@ -35,3 +37,41 @@ static inline void print_with_escapes(const unsigned char *src) {
}
}
static inline void print_with_escapes(const unsigned char *src, std::ostream &os) {
while (*src) {
switch (*src) {
case '\n':
os << '\\';
os << 'n';
break;
case '\r':
os << '\\';
os << 'r';
break;
case '\"':
os << '\\';
os << '"';
break;
case '\t':
os << '\\';
os << 't';
break;
case '\\':
os << '\\';
os << '\\';
break;
default:
if (*src <= 0x1F) {
std::ios::fmtflags f(os.flags());
os << std::hex << std::setw(4) << std::setfill('0') << (int) *src;
os.flags(f);
} else
os << *src;
}
src++;
}
}
static inline void print_with_escapes(const char *src, std::ostream &os) {
print_with_escapes((const unsigned char *)src, os);
}
include/simdjson/parsedjson.h
+27 -11
View File
@@ -264,16 +264,16 @@ public:
printf("false\n");
break;
case '{': // we have an object
printf("{\n");
printf("{\t// pointing to next tape location %llu \n", payload);
break;
case '}': // we end an object
printf("}\n");
printf("}\t// pointing to previous tape location %llu \n", payload);
break;
case '[': // we start an array
printf("[\n");
printf("[\t// pointing to next tape location %llu \n", payload);
break;
case ']': // we end an array
printf("]\n");
printf("]\t// pointing to previous tape location %llu \n", payload);
break;
case 'r': // we start and end with the root node
printf("end of root\n");
@@ -368,7 +368,7 @@ public:
current_val(o.current_val), depthindex(o.depthindex) {
o.depthindex = NULL;// we take ownship
}
WARN_UNUSED
bool isOk() const {
return location < tape_length;
}
@@ -384,7 +384,9 @@ public:
// return true if we can do the navigation, false
// otherwise
// valid if we're not at the end of a scope
// withing a give scope, we move forward
// valid if we're not at the end of a scope (returns true)
WARN_UNUSED
really_inline bool next() {
if ((current_type == '[') || (current_type == '{')){
// we need to jump
@@ -417,6 +419,7 @@ public:
}
// valid if we're not at the start of a scope
WARN_UNUSED
really_inline bool prev() {
if(location - 1 < depthindex[depth]) return false;
location -= 1;
@@ -437,6 +440,7 @@ public:
// valid unless we are at the first level of the document
WARN_UNUSED
really_inline bool up() {
if(depth == 1) {
return false; // don't allow moving back to root
@@ -451,11 +455,16 @@ public:
}
// valid if we're at a [ or { call site; moves us to start of
// that scope
// valid if we're at a [ or { and it starts a non-empty scope; moves us to start of
// that deeper scope if it not empty
WARN_UNUSED
really_inline bool down() {
if(location + 1 >= tape_length) return false;
if ((current_type == '[') || (current_type == '{')) {
size_t npos = (current_val & JSONVALUEMASK);
if(npos == location + 2) {
return false; // we have an empty scope
}
depth++;
location = location + 1;
depthindex[depth] = location;
@@ -477,17 +486,24 @@ public:
// the start of our current scope; always succeeds
// print the thing we're currently pointing at
bool print(std::ostream &os) const {
bool print(std::ostream &os, bool escape_strings = true) const {
if(!isOk()) return false;
switch (current_type) {
case '"': // we have a string
os << '"' << get_string() << '"';
os << '"';
if(escape_strings) {
print_with_escapes(get_string(), os);
} else {
os << get_string();
}
os << '"';
break;
case 'l': // we have a long int
os << get_integer();
break;
case 'd':
os << get_double();
os << get_double();
break;
case 'n': // we have a null
os << "null";
break;
scripts/testjson2json.sh
+61 -2
View File
@@ -2,9 +2,10 @@
TMPDIR1=$(mktemp -d -t simdjsonXXXXXXXX)
TMPDIR2=$(mktemp -d -t simdjsonXXXXXXXX)
TMPDIR3=$(mktemp -d -t simdjsonXXXXXXXX)
TMPDIR4=$(mktemp -d -t simdjsonXXXXXXXX)
trap "exit 1" HUP INT PIPE QUIT TERM
trap "rm -rf $TMPDIR1 $TMPDIR2" EXIT
trap "rm -rf $TMPDIR1 $TMPDIR2 $TMPDIR3 $TMPDIR4" EXIT
function founderror() {
echo "code is wrong"
@@ -15,19 +16,33 @@ make minify json2json
for i in `cd jsonexamples && ls -1 *.json`; do
echo $i
./json2json jsonexamples/$i > $TMPDIR1/$i
./json2json -a jsonexamples/$i > $TMPDIR3/$i
./json2json $TMPDIR1/$i > $TMPDIR2/$i
./json2json -a $TMPDIR3/$i > $TMPDIR4/$i
cmp $TMPDIR1/$i $TMPDIR2/$i
retVal=$?
if [ $retVal -ne 0 ]; then
founderror
fi
cmp $TMPDIR3/$i $TMPDIR4/$i
retVal=$?
if [ $retVal -ne 0 ]; then
founderror
fi
./minify $TMPDIR1/$i > $TMPDIR1/minify$i
./minify $TMPDIR2/$i > $TMPDIR2/minify$i
./minify $TMPDIR3/$i > $TMPDIR3/minify$i
./minify $TMPDIR4/$i > $TMPDIR4/minify$i
cmp $TMPDIR1/minify$i $TMPDIR2/minify$i
retVal=$?
if [ $retVal -ne 0 ]; then
founderror
fi
cmp $TMPDIR3/minify$i $TMPDIR4/minify$i
retVal=$?
if [ $retVal -ne 0 ]; then
founderror
fi
./json2json $TMPDIR1/minify$i > $TMPDIR2/bisminify$i
cmp $TMPDIR1/$i $TMPDIR2/bisminify$i
retVal=$?
@@ -35,6 +50,50 @@ for i in `cd jsonexamples && ls -1 *.json`; do
founderror
fi
done
for i in `cd jsonchecker && ls -1 pass*.json`; do
echo $i
./json2json jsonchecker/$i > $TMPDIR1/$i
./json2json -a jsonchecker/$i > $TMPDIR3/$i
./json2json $TMPDIR1/$i > $TMPDIR2/$i
./json2json -a $TMPDIR3/$i > $TMPDIR4/$i
cmp $TMPDIR1/$i $TMPDIR2/$i
retVal=$?
if [ $retVal -ne 0 ]; then
echo "reg failure"
founderror
fi
cmp $TMPDIR3/$i $TMPDIR4/$i
retVal=$?
if [ $retVal -ne 0 ]; then
echo "-a failure"
founderror
fi
./minify $TMPDIR1/$i > $TMPDIR1/minify$i
./minify $TMPDIR2/$i > $TMPDIR2/minify$i
./minify $TMPDIR3/$i > $TMPDIR3/minify$i
./minify $TMPDIR4/$i > $TMPDIR4/minify$i
cmp $TMPDIR1/minify$i $TMPDIR2/minify$i
retVal=$?
if [ $retVal -ne 0 ]; then
echo "reg failure, step 2"
founderror
fi
cmp $TMPDIR3/minify$i $TMPDIR4/minify$i
retVal=$?
if [ $retVal -ne 0 ]; then
echo "-a failure, step 2"
founderror
fi
./json2json $TMPDIR1/minify$i > $TMPDIR2/bisminify$i
cmp $TMPDIR1/$i $TMPDIR2/bisminify$i
retVal=$?
if [ $retVal -ne 0 ]; then
founderror
fi
done
echo "test successful"
exit 0
tools/json2json.cpp
+9 -2
View File
@@ -14,7 +14,14 @@ void compute_dump(ParsedJson::iterator &pjh) {
return; // we are done
}
// we have either an array or an object
pjh.down();
bool goingdown = pjh.down();
if(!goingdown) {
// we have an empty scope
if(inobject) std::cout<<"{}";
else std::cout<<"[]";
return;
}
// we have a non-empty scope and we are at the beginning of it
if (inobject) {
std::cout << "{";
assert(pjh.get_type() == '"');
@@ -40,7 +47,7 @@ void compute_dump(ParsedJson::iterator &pjh) {
}
std::cout << "]";
}
pjh.up();
assert(pjh.up());
}
int main(int argc, char *argv[]) {