From 7fe52e0fbe87246d1316435d183b90fda761d013 Mon Sep 17 00:00:00 2001
From: Daniel Lemire <lemire@gmail.com>
Date: Sun, 29 Jul 2018 13:09:47 -0400
Subject: [PATCH] Histograms and more widths

---
 Makefile | 17 +++++++++++++++--
 main.cpp | 16 ++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index d16e41de7..903793b4b 100644
--- a/Makefile
+++ b/Makefile
@@ -6,7 +6,7 @@
 
 .PHONY: clean cleandist
 
-CXXFLAGS =  -std=c++11 -O2 -march=native -Wall -Wextra -Wshadow -Idependencies/double-conversion -Ldependencies/double-conversion/release 
+CXXFLAGS =  -std=c++11 -O2 -march=native -Wall -Wextra -Wshadow -Idependencies/double-conversion -Ldependencies/double-conversion/release
 LIBFLAGS = -ldouble-conversion
 #CXXFLAGS =  -std=c++11 -O2 -march=native -Wall -Wextra -Wshadow -Wno-implicit-function-declaration
 
@@ -27,13 +27,18 @@ $(LIDDOUBLE) : dependencies/double-conversion/README.md
 parse: main.cpp common_defs.h linux-perf-events.h
 	$(CXX) $(CXXFLAGS) -o parse main.cpp $(LIBFLAGS)
 
-testflatten: parse parsenocheesy parsenodep8
+parsehisto: main.cpp common_defs.h linux-perf-events.h
+	$(CXX) $(CXXFLAGS) -o parsehisto main.cpp $(LIBFLAGS) -DBUILDHISTOGRAM
+
+testflatten: parse parsenocheesy parsenodep8 parsenodep10 parsenodep12
 	for filename in jsonexamples/twitter.json jsonexamples/gsoc-2018.json jsonexamples/citm_catalog.json jsonexamples/canada.json ; do \
         	echo $$filename ; \
 		set -x; \
 		./parsenocheesy $$filename ; \
 		./parse $$filename ; \
 		./parsenodep8 $$filename ; \
+		./parsenodep10 $$filename ; \
+		./parsenodep12 $$filename ; \
 		set +x; \
 	done
 
@@ -43,6 +48,14 @@ parsenocheesy: main.cpp common_defs.h linux-perf-events.h
 parsenodep8: main.cpp common_defs.h linux-perf-events.h
 	$(CXX) $(CXXFLAGS) -o parsenodep8 main.cpp -DNO_PDEP_PLEASE -DNO_PDEP_WIDTH=8
 
+parsenodep10: main.cpp common_defs.h linux-perf-events.h
+	$(CXX) $(CXXFLAGS) -o parsenodep12 main.cpp -DNO_PDEP_PLEASE -DNO_PDEP_WIDTH=10
+
+
+parsenodep12: main.cpp common_defs.h linux-perf-events.h
+	$(CXX) $(CXXFLAGS) -o parsenodep12 main.cpp -DNO_PDEP_PLEASE -DNO_PDEP_WIDTH=12
+
+
 dependencies/double-conversion/README.md:
 	git pull && git submodule init && git submodule update && git submodule status
 
diff --git a/main.cpp b/main.cpp
index 59b204aae..f5cf6b75c 100644
--- a/main.cpp
+++ b/main.cpp
@@ -328,6 +328,22 @@ never_inline bool flatten_indexes(size_t len, ParsedJson & pj) {
     u32 * base_ptr = pj.structural_indexes;
     base_ptr[DUMMY_NODE] = base_ptr[ROOT_NODE] = 0; // really shouldn't matter
     u32 base = NUM_RESERVED_NODES;
+#ifdef BUILDHISTOGRAM
+    uint32_t counters [65];
+    uint32_t total = 0;
+    for(int k = 0; k < 66; k++) counters[k] = 0;
+    for (size_t idx = 0; idx < len; idx+=64) {
+        u64 s = *(u64 *)(pj.structurals + idx/8);
+        u32 cnt = __builtin_popcountll(s);
+        total ++;
+        counters[cnt]++;
+    }
+    printf("\n histogram:\n");
+    for(int k = 0; k < 66; k++) {
+      if(counters[k]>0)printf("%10d %10.u %10.3f \n", k, counters[k], counters[k] * 1.0 / total);
+    }
+    printf("\n\n");
+#endif
     for (size_t idx = 0; idx < len; idx+=64) {
         u64 s = *(u64 *)(pj.structurals + idx/8);
 #ifdef SUPPRESS_CHEESY_FLATTEN