admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Histograms and more widths

Browse Source
This commit is contained in:
Daniel Lemire
2018-07-29 13:09:47 -04:00
parent 96991e7623
commit 7fe52e0fbe
2 changed files with 31 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+15 -2
View File
@@ -6,7 +6,7 @@
.PHONY: clean cleandist
CXXFLAGS = -std=c++11 -O2 -march=native -Wall -Wextra -Wshadow -Idependencies/double-conversion -Ldependencies/double-conversion/release
CXXFLAGS = -std=c++11 -O2 -march=native -Wall -Wextra -Wshadow -Idependencies/double-conversion -Ldependencies/double-conversion/release
LIBFLAGS = -ldouble-conversion
#CXXFLAGS = -std=c++11 -O2 -march=native -Wall -Wextra -Wshadow -Wno-implicit-function-declaration
@@ -27,13 +27,18 @@ $(LIDDOUBLE) : dependencies/double-conversion/README.md
parse: main.cpp common_defs.h linux-perf-events.h
$(CXX) $(CXXFLAGS) -o parse main.cpp $(LIBFLAGS)
testflatten: parse parsenocheesy parsenodep8
parsehisto: main.cpp common_defs.h linux-perf-events.h
$(CXX) $(CXXFLAGS) -o parsehisto main.cpp $(LIBFLAGS) -DBUILDHISTOGRAM
testflatten: parse parsenocheesy parsenodep8 parsenodep10 parsenodep12
for filename in jsonexamples/twitter.json jsonexamples/gsoc-2018.json jsonexamples/citm_catalog.json jsonexamples/canada.json ; do \
echo $$filename ; \
set -x; \
./parsenocheesy $$filename ; \
./parse $$filename ; \
./parsenodep8 $$filename ; \
./parsenodep10 $$filename ; \
./parsenodep12 $$filename ; \
set +x; \
done
@@ -43,6 +48,14 @@ parsenocheesy: main.cpp common_defs.h linux-perf-events.h
parsenodep8: main.cpp common_defs.h linux-perf-events.h
$(CXX) $(CXXFLAGS) -o parsenodep8 main.cpp -DNO_PDEP_PLEASE -DNO_PDEP_WIDTH=8
parsenodep10: main.cpp common_defs.h linux-perf-events.h
$(CXX) $(CXXFLAGS) -o parsenodep12 main.cpp -DNO_PDEP_PLEASE -DNO_PDEP_WIDTH=10
parsenodep12: main.cpp common_defs.h linux-perf-events.h
$(CXX) $(CXXFLAGS) -o parsenodep12 main.cpp -DNO_PDEP_PLEASE -DNO_PDEP_WIDTH=12
dependencies/double-conversion/README.md:
git pull && git submodule init && git submodule update && git submodule status
main.cpp
+16
View File
@@ -328,6 +328,22 @@ never_inline bool flatten_indexes(size_t len, ParsedJson & pj) {
u32 * base_ptr = pj.structural_indexes;
base_ptr[DUMMY_NODE] = base_ptr[ROOT_NODE] = 0; // really shouldn't matter
u32 base = NUM_RESERVED_NODES;
#ifdef BUILDHISTOGRAM
uint32_t counters [65];
uint32_t total = 0;
for(int k = 0; k < 66; k++) counters[k] = 0;
for (size_t idx = 0; idx < len; idx+=64) {
u64 s = *(u64 *)(pj.structurals + idx/8);
u32 cnt = __builtin_popcountll(s);
total ++;
counters[cnt]++;
}
printf("\n histogram:\n");
for(int k = 0; k < 66; k++) {
if(counters[k]>0)printf("%10d %10.u %10.3f \n", k, counters[k], counters[k] * 1.0 / total);
}
printf("\n\n");
#endif
for (size_t idx = 0; idx < len; idx+=64) {
u64 s = *(u64 *)(pj.structurals + idx/8);
#ifdef SUPPRESS_CHEESY_FLATTEN