Producing a new release

* Hiding the runtime dispatch pointer in a source file so it is not an exported symbol
* Disabling hard failure on style check.
* Fixes https://github.com/lemire/simdjson/issues/250

.appveyor.yml

@@ -9,9 +9,14 @@ clone_folder: c:\projects\simdjson
 platform:
 - x64
 
+environment:
+  matrix:
+    - SIMDJSON_BUILD_STATIC: "OFF"
+    - SIMDJSON_BUILD_STATIC: "ON"
+      
 build_script:
   - mkdir build
   - cd build
-  - ps: cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..
+  - ps: cmake -DSIMDJSON_BUILD_STATIC="$env:SIMDJSON_BUILD_STATIC" -DCMAKE_BUILD_TYPE=Release -DCMAKE_GENERATOR_PLATFORM=x64 ..
   - cmake --build .
   - ctest --verbose

.circleci/config.yml

@@ -14,6 +14,7 @@ jobs:
           build-essential
           cmake
           g++-7
+          git
 
       - run:
           name: Building (gcc)
@@ -37,6 +38,44 @@ jobs:
             cd build
             make test
 
+  "gccnoavx":
+    docker:
+      - image: ubuntu:18.04
+        environment:
+          CXX: g++-7
+    steps:
+      - checkout
+
+      - run: apt-get update -qq
+      - run: >
+          apt-get install -y
+          build-essential
+          cmake
+          g++-7
+          git
+
+      - run:
+          name: Building (gcc)
+          command: ARCHFLAGS="-march=nehalem" make
+
+      - run:
+          name: Running tests (gcc)
+          command: ARCHFLAGS="-march=nehalem" make quiettest amalgamate
+
+      - run:
+          name: Building (gcc, cmake)
+          command: |
+            mkdir build
+            cd build
+            cmake -DSIMDJSON_DISABLE_AVX=on ..
+            make
+
+      - run:
+          name: Running tests (gcc, cmake)
+          command: |
+            cd build
+            make test
+
   "clang":
     docker:
       - image: ubuntu:18.04
@@ -51,6 +90,7 @@ jobs:
           build-essential
           cmake
           clang-6.0
+          git
 
       - run:
           name: Building (clang)
@@ -74,9 +114,49 @@ jobs:
             cd build
             make test
 
+  "clangnoavx":
+    docker:
+      - image: ubuntu:18.04
+        environment:
+          CXX: clang++-6.0
+    steps:
+      - checkout
+
+      - run: apt-get update -qq
+      - run: >
+          apt-get install -y
+          build-essential
+          cmake
+          clang-6.0
+          git
+
+      - run:
+          name: Building (clang)
+          command: ARCHFLAGS="-march=nehalem"  make
+
+      - run:
+          name: Running tests (clang)
+          command: ARCHFLAGS="-march=nehalem"  make quiettest amalgamate
+
+      - run:
+          name: Building (clang, cmake)
+          command: |
+            mkdir build
+            cd build
+            cmake -DSIMDJSON_DISABLE_AVX=on ..
+            make
+
+      - run:
+          name: Running tests (clang, cmake)
+          command: |
+            cd build
+            make test
+
 workflows:
   version: 2
   build_and_test:
     jobs:
       - "clang"
       - "gcc"
+      - "clangnoavx"
+      - "gccnoavx"

.clang-format

@@ -0,0 +1 @@
+BasedOnStyle: LLVM

.drone.yml

@@ -1,10 +1,234 @@
 kind: pipeline
-name: default
+name: x64
+
+platform:
+  os: linux
+  arch: amd64
 
 steps:
 - name: test
   image: gcc:8
   commands:
-  - make -j2
-  - make quiettest -j2
+  - make
+  - make quiettest
   - make amalgamate
+---
+kind: pipeline
+name: arm64
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+- name: test
+  image: gcc:8
+  commands:
+  - make
+  - make quiettest
+  - make amalgamate
+---
+kind: pipeline
+name: stylecheck
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: Build and Test
+  image: ubuntu:18.04
+  commands:
+    - apt-get update -y
+    - apt-get install -y python clang-format
+    - ./style/run-clang-format.py -r include/ benchmark/ src/ tests/
+---
+kind: pipeline
+name: amd64_clang_cmake_dynamic
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: Build and Test
+  image: ubuntu:18.04
+  environment:
+    CC: clang
+    CXX: clang++
+  commands:
+    - apt-get update -y
+    - apt-get install -y make $CC g++ cmake 
+    - $CC --version
+    - mkdir build && cd build
+    - cmake -DSIMDJSON_BUILD_STATIC=OFF $CMAKE_FLAGS ..
+    - make -j
+    - ctest
+---
+kind: pipeline
+name: amd64_clang_cmake_static
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: Build and Test
+  image: ubuntu:18.04
+  environment:
+    CC: clang
+    CXX: clang++
+  commands:
+    - apt-get update -y
+    - apt-get install -y make $CC g++ cmake 
+    - $CC --version
+    - mkdir build && cd build
+    - cmake -DSIMDJSON_BUILD_STATIC=ON $CMAKE_FLAGS ..
+    - make -j
+    - ctest
+---
+kind: pipeline
+name: amd64_gcc_cmake_static
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: Build and Test
+  image: ubuntu:18.04
+  environment:
+    CC: gcc
+    CXX: g++
+  commands:
+    - apt-get update -y
+    - apt-get install -y make $CC g++ cmake 
+    - $CC --version
+    - mkdir build && cd build
+    - cmake -DSIMDJSON_BUILD_STATIC=ON $CMAKE_FLAGS ..
+    - make -j
+    - ctest
+---
+kind: pipeline
+name: amd64_gcc_cmake_dynamic
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: Build and Test
+  image: ubuntu:18.04
+  environment:
+    CC: gcc
+    CXX: g++
+  commands:
+    - apt-get update -y
+    - apt-get install -y make $CC g++ cmake 
+    - $CC --version
+    - mkdir build && cd build
+    - cmake -DSIMDJSON_BUILD_STATIC=OFF $CMAKE_FLAGS ..
+    - make -j
+    - ctest
+---
+kind: pipeline
+name: arm64_clang_cmake_dynamic
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+- name: Build and Test
+  image: ubuntu:18.04
+  environment:
+    CC: clang
+    CXX: clang++
+  commands:
+    - apt-get update -y
+    - apt-get install -y make $CC g++ cmake 
+    - $CC --version
+    - mkdir build && cd build
+    - cmake -DSIMDJSON_BUILD_STATIC=OFF $CMAKE_FLAGS ..
+    - make -j
+    - ctest
+---
+kind: pipeline
+name: arm64_gcc_cmake_dynamic
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+- name: Build and Test
+  image: ubuntu:18.04
+  environment:
+    CC: gcc
+    CXX: g++
+  commands:
+    - apt-get update -y
+    - apt-get install -y make $CC g++ cmake
+    - $CC --version
+    - mkdir build && cd build
+    - cmake $CMAKE_FLAGS ..
+    - make -j
+    - ctest
+
+  commands:
+    - apt-get update -y
+    - apt-get install -y make $CC g++ cmake 
+    - $CC --version
+    - mkdir build && cd build
+    - cmake -DSIMDJSON_BUILD_STATIC=OFF $CMAKE_FLAGS ..
+    - make -j
+    - ctest
+---
+kind: pipeline
+name: arm64_clang_cmake_static
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+- name: Build and Test
+  image: ubuntu:18.04
+  environment:
+    CC: clang
+    CXX: clang++
+  commands:
+    - apt-get update -y
+    - apt-get install -y make $CC g++ cmake 
+    - $CC --version
+    - mkdir build && cd build
+    - cmake -DSIMDJSON_BUILD_STATIC=ON $CMAKE_FLAGS ..
+    - make -j
+    - ctest
+---
+kind: pipeline
+name: arm64_gcc_cmake_static
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+- name: Build and Test
+  image: ubuntu:18.04
+  environment:
+    CC: gcc
+    CXX: g++
+  commands:
+    - apt-get update -y
+    - apt-get install -y make $CC g++ cmake
+    - $CC --version
+    - mkdir build && cd build
+    - cmake -DSIMDJSON_BUILD_STATIC=ON $CMAKE_FLAGS ..
+    - make -j
+    - ctest
+
+
+
+
+

.gitmodules

@@ -25,3 +25,6 @@
 [submodule "dependencies/jsoncpp"]
 	path = dependencies/jsoncpp
 	url = https://github.com/open-source-parsers/jsoncpp.git
+[submodule "dependencies/json"]
+	path = dependencies/json
+	url = https://github.com/nlohmann/json.git

.travis.yml

@@ -7,7 +7,8 @@ addons:
     packages:
       - gcc-7
       - g++-7
-
+      - clang-format
+      - python
 branches:
   only:
     - master
@@ -17,3 +18,13 @@ script:
    - export CC=gcc-7
    - make
    - make test 
+   - make everything
+   - make amalgamate
+   - make clean
+   - make SANITIZEGOLD=1 test
+   - make clean
+   - ARCHFLAGS="-march=nehalem" make 
+   - ARCHFLAGS="-march=nehalem" make test
+   - ARCHFLAGS="-march=nehalem" make everything
+   - ./style/run-clang-format.py -r include/ benchmark/ src/ tests/
+

CMakeLists.txt

@@ -1,4 +1,13 @@
-cmake_minimum_required(VERSION 3.8...3.13)
+cmake_minimum_required(VERSION 3.9) # CMP0069 NEW
+include(CheckIPOSupported)
+check_ipo_supported(RESULT ltoresult)
+if(ltoresult)
+set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
+endif()
+
+# usage: cmake -DSIMDJSON_DISABLE_AVX=on ..
+option(SIMDJSON_DISABLE_AVX "Forcefully disable AVX even if hardware supports it" OFF)
+
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_MACOSX_RPATH OFF)
@@ -10,9 +19,9 @@ endif()
 project(simdjson)
 set(SIMDJSON_LIB_NAME simdjson)
 set(PROJECT_VERSION_MAJOR 0)
-set(PROJECT_VERSION_MINOR 1)
-set(PROJECT_VERSION_PATCH 0)
-set(SIMDJSON_LIB_VERSION "0.1.0" CACHE STRING "simdjson library version")
+set(PROJECT_VERSION_MINOR 2)
+set(PROJECT_VERSION_PATCH 1)
+set(SIMDJSON_LIB_VERSION "0.2.1" CACHE STRING "simdjson library version")
 set(SIMDJSON_LIB_SOVERSION "0" CACHE STRING "simdjson library soversion")
 
 if(NOT MSVC)
@@ -20,14 +29,12 @@ option(SIMDJSON_BUILD_STATIC "Build a static library" OFF) # turning it on disab
 else()
 option(SIMDJSON_BUILD_STATIC "Build a static library" ON) # turning it on disables the production of a dynamic library
 endif()
-option(SIMDJSON_BUILD_LTO "Build library with Link Time Optimization" OFF)
 option(SIMDJSON_SANITIZE "Sanitize addresses" OFF)
 
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake")
 
 find_package(CTargets)
 find_package(Options)
-find_package(LTO)
 
 install(DIRECTORY include/${SIMDJSON_LIB_NAME} DESTINATION include)
 set (TEST_DATA_DIR "${CMAKE_CURRENT_SOURCE_DIR}/jsonchecker/")

CONTRIBUTORS

@@ -0,0 +1,27 @@
+# contributors (in no particular order)
+Thomas Navennec
+Kai Wolf
+Tyler Kennedy
+Frank Wessels
+George Fotopoulos
+Heinz N. Gies
+Emil Gedda
+Wojciech Muła
+Georgios Floros
+Dong Xie
+Nan Xiao
+Egor Bogatov
+Jinxi Wang
+Luiz Fernando Peres
+Wouter Bolsterlee
+Anish Karandikar
+Reini Urban
+Tom Dyson
+Ihor Dotsenko
+Alexey Milovidov
+Chang Liu
+Sunny Gleason
+John Keiser
+Zach Bjornson
+# if you have contributed to the project and your name does not 
+# appear in this list, please let us know!

Dockerfile

@@ -0,0 +1,10 @@
+# docker build -t simdjson . && docker run --privileged -t simdjson
+FROM gcc:8.3
+COPY . /usr/src/
+WORKDIR /usr/src/
+RUN make clean
+RUN make amalgamate
+RUN make
+RUN make test
+RUN make parsingcompetition
+CMD ["bash", "scripts/selectparser.sh"]

Makefile

@@ -5,30 +5,65 @@
 
 
 .PHONY: clean cleandist
-COREDEPSINCLUDE = -Idependencies/rapidjson/include -Idependencies/sajson/include -Idependencies/cJSON  -Idependencies/jsmn
+COREDEPSINCLUDE = -Idependencies/json/single_include -Idependencies/rapidjson/include -Idependencies/sajson/include -Idependencies/cJSON  -Idependencies/jsmn
 EXTRADEPSINCLUDE =  -Idependencies/jsoncppdist -Idependencies/json11 -Idependencies/fastjson/src -Idependencies/fastjson/include -Idependencies/gason/src -Idependencies/ujson4c/3rdparty -Idependencies/ujson4c/src
-CXXFLAGS =  -std=c++17  -march=native -Wall -Wextra -Wshadow -Iinclude  -Ibenchmark/linux
-CFLAGS = -march=native  -Idependencies/ujson4c/3rdparty -Idependencies/ujson4c/src
+# users can provide their own additional flags with make EXTRAFLAGS=something
+architecture:=$(shell arch)
+
+####
+# If you want to specify your own target architecture,
+# then define ARCHFLAGS. Otherwise, we set good default.
+# E.g., type ' ARCHFLAGS="-march=nehalem" make parse '
+###
+ifeq ($(architecture),aarch64)
+ARCHFLAGS ?= -march=armv8-a+crc+crypto
+else
+ARCHFLAGS ?= -msse4.2 -mpclmul # lowest supported feature set?
+endif
+
+CXXFLAGS = $(ARCHFLAGS) -std=c++17   -Wall -Wextra -Wshadow -Iinclude  -Ibenchmark/linux $(EXTRAFLAGS)
+CFLAGS =  $(ARCHFLAGS)  -Idependencies/ujson4c/3rdparty -Idependencies/ujson4c/src $(EXTRAFLAGS)
+
+
+# This is a convenience flag
+ifdef SANITIZEGOLD
+    SANITIZE = 1
+    LINKER = gold
+endif
+
+ifdef LINKER
+	CXXFLAGS += -fuse-ld=$(LINKER)
+	CFLAGS += -fuse-ld=$(LINKER)
+endif
+
+
+# SANITIZE *implies* DEBUG
+ifeq ($(MEMSANITIZE),1)
+        CXXFLAGS += -g3 -O0  -fsanitize=memory -fno-omit-frame-pointer -fsanitize=undefined
+        CFLAGS += -g3 -O0  -fsanitize=memory -fno-omit-frame-pointer -fsanitize=undefined
+else
 ifeq ($(SANITIZE),1)
-	CXXFLAGS += -g3 -O0  -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined
-	CFLAGS += -g3 -O0  -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined
+	CXXFLAGS += -g3 -O0  -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined 
+	CFLAGS += -g3 -O0  -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined 
 else
 ifeq ($(DEBUG),1)
         CXXFLAGS += -g3 -O0
         CFLAGS += -g3 -O0
 else
+# we opt for  -O3 for regular builds
 	CXXFLAGS += -O3
 	CFLAGS += -O3
-endif
-endif
+endif # ifeq ($(DEBUG),1)
+endif # ifeq ($(SANITIZE),1)
+endif # ifeq ($(MEMSANITIZE),1)
 
-MAINEXECUTABLES=parse minify json2json jsonstats statisticalmodel
-TESTEXECUTABLES=jsoncheck numberparsingcheck stringparsingcheck
+MAINEXECUTABLES=parse minify json2json jsonstats statisticalmodel jsonpointer
+TESTEXECUTABLES=jsoncheck numberparsingcheck stringparsingcheck pointercheck
 COMPARISONEXECUTABLES=minifiercompetition parsingcompetition parseandstatcompetition distinctuseridcompetition allparserscheckfile allparsingcompetition
 SUPPLEMENTARYEXECUTABLES=parse_noutf8validation parse_nonumberparsing parse_nostringparsing
 
-HEADERS= include/simdjson/simdutf8check.h include/simdjson/stringparsing.h include/simdjson/numberparsing.h include/simdjson/jsonparser.h include/simdjson/common_defs.h include/simdjson/jsonioutil.h benchmark/benchmark.h benchmark/linux/linux-perf-events.h include/simdjson/parsedjson.h include/simdjson/stage1_find_marks.h include/simdjson/stage2_build_tape.h include/simdjson/jsoncharutils.h include/simdjson/jsonformatutils.h
-LIBFILES=src/jsonioutil.cpp src/jsonparser.cpp src/simdjson.cpp src/stage1_find_marks.cpp        src/stage2_build_tape.cpp src/parsedjson.cpp src/parsedjsoniterator.cpp
+HEADERS= include/simdjson/simdutf8check_haswell.h include/simdjson/simdutf8check_westmere.h include/simdjson/simdutf8check_arm64.h include/simdjson/stringparsing.h include/simdjson/stringparsing_arm64.h  include/simdjson/stringparsing_haswell.h  include/simdjson/stringparsing_macros.h  include/simdjson/stringparsing_westmere.h include/simdjson/numberparsing.h include/simdjson/jsonparser.h include/simdjson/common_defs.h include/simdjson/jsonioutil.h benchmark/benchmark.h benchmark/linux/linux-perf-events.h include/simdjson/parsedjson.h include/simdjson/stage1_find_marks.h  include/simdjson/stage1_find_marks_arm64.h  include/simdjson/stage1_find_marks_haswell.h   include/simdjson/stage1_find_marks_westmere.h   include/simdjson/stage1_find_marks_macros.h include/simdjson/stage2_build_tape.h include/simdjson/jsoncharutils.h include/simdjson/jsonformatutils.h include/simdjson/stage1_find_marks_flatten.h include/simdjson/stage1_find_marks_flatten_haswell.h
+LIBFILES=src/jsonioutil.cpp src/jsonparser.cpp src/simdjson.cpp src/stage1_find_marks.cpp src/stage2_build_tape.cpp src/parsedjson.cpp src/parsedjsoniterator.cpp
 MINIFIERHEADERS=include/simdjson/jsonminifier.h include/simdjson/simdprune_tables.h
 MINIFIERLIBFILES=src/jsonminifier.cpp
 
@@ -41,9 +76,9 @@ GASON_INCLUDE:=dependencies/gason/src/gason.h
 UJSON4C_INCLUDE:=dependencies/ujson4c/src/ujdecode.c
 CJSON_INCLUDE:=dependencies/cJSON/cJSON.h
 JSMN_INCLUDE:=dependencies/jsmn/jsmn.h
+JSON_INCLUDE:=dependencies/json/single_include/nlohmann/json.hpp
 
-
-LIBS=$(RAPIDJSON_INCLUDE) $(SAJSON_INCLUDE) $(JSON11_INCLUDE) $(FASTJSON_INCLUDE) $(GASON_INCLUDE) $(UJSON4C_INCLUDE) $(CJSON_INCLUDE) $(JSMN_INCLUDE)
+LIBS=$(RAPIDJSON_INCLUDE) $(JSON_INCLUDE) $(SAJSON_INCLUDE) $(JSON11_INCLUDE) $(FASTJSON_INCLUDE) $(GASON_INCLUDE) $(UJSON4C_INCLUDE) $(CJSON_INCLUDE) $(JSMN_INCLUDE)
 
 EXTRAOBJECTS=ujdecode.o
 all:  $(MAINEXECUTABLES)
@@ -56,44 +91,34 @@ benchmark:
 	bash ./scripts/parser.sh
 	bash ./scripts/parseandstat.sh
 
-test: jsoncheck numberparsingcheck stringparsingcheck
+test: jsoncheck numberparsingcheck stringparsingcheck basictests allparserscheckfile minify json2json pointercheck
+	./basictests
 	./numberparsingcheck
 	./stringparsingcheck
 	./jsoncheck
+	./pointercheck
 	./scripts/testjson2json.sh
-	@echo
-	@tput setaf 2
+	./scripts/issue150.sh
 	@echo "It looks like the code is good!"
-	@tput sgr0
 
-quiettest: jsoncheck numberparsingcheck stringparsingcheck
+quiettest: jsoncheck numberparsingcheck stringparsingcheck basictests allparserscheckfile minify json2json pointercheck
+	./basictests
 	./numberparsingcheck
 	./stringparsingcheck
 	./jsoncheck
+	./pointercheck
 	./scripts/testjson2json.sh
+	./scripts/issue150.sh
 
 amalgamate:
 	./amalgamation.sh
+	$(CXX) $(CXXFLAGS) -o singleheader/demo  ./singleheader/amalgamation_demo.cpp   -Isingleheader   
 
-$(SAJSON_INCLUDE):
-	git submodule update --init --recursive
-
-$(RAPIDJSON_INCLUDE):
-	git submodule update --init --recursive
-
-$(JSON11_INCLUDE):
-	git submodule update --init --recursive
-
-$(FASTJSON_INCLUDE):
-	git submodule update --init --recursive
-
-$(GASON_INCLUDE):
-	git submodule update --init --recursive
-
-$(UJSON4C_INCLUDE):
-	git submodule update --init --recursive
-
+submodules: 
+	-git submodule update --init --recursive
+	-touch submodules
 
+$(JSON_INCLUDE) $(SAJSON_INCLUDE) $(RAPIDJSON_INCLUDE) $(JSON11_INCLUDE) $(FASTJSON_INCLUDE) $(GASON_INCLUDE) $(UJSON4C_INCLUDE) $(CJSON_INCLUDE) $(JSMN_INCLUDE) : submodules
 
 parse: benchmark/parse.cpp $(HEADERS) $(LIBFILES)
 	$(CXX) $(CXXFLAGS) -o parse $(LIBFILES) benchmark/parse.cpp $(LIBFLAGS)
@@ -115,6 +140,10 @@ parse_nostringparsing: benchmark/parse.cpp $(HEADERS) $(LIBFILES)
 jsoncheck:tests/jsoncheck.cpp $(HEADERS) $(LIBFILES)
 	$(CXX) $(CXXFLAGS) -o jsoncheck $(LIBFILES) tests/jsoncheck.cpp -I. $(LIBFLAGS)
 
+basictests:tests/basictests.cpp $(HEADERS) $(LIBFILES)
+	$(CXX) $(CXXFLAGS) -o basictests $(LIBFILES) tests/basictests.cpp -I. $(LIBFLAGS)
+
+
 numberparsingcheck:tests/numberparsingcheck.cpp $(HEADERS) $(LIBFILES)
 	$(CXX) $(CXXFLAGS) -o numberparsingcheck tests/numberparsingcheck.cpp  src/jsonioutil.cpp src/jsonparser.cpp src/simdjson.cpp src/stage1_find_marks.cpp  src/parsedjson.cpp       -I. $(LIBFLAGS) -DJSON_TEST_NUMBERS
 
@@ -122,8 +151,10 @@ numberparsingcheck:tests/numberparsingcheck.cpp $(HEADERS) $(LIBFILES)
 stringparsingcheck:tests/stringparsingcheck.cpp $(HEADERS) $(LIBFILES)
 	$(CXX) $(CXXFLAGS) -o stringparsingcheck tests/stringparsingcheck.cpp  src/jsonioutil.cpp src/jsonparser.cpp src/simdjson.cpp src/stage1_find_marks.cpp  src/parsedjson.cpp      -I. $(LIBFLAGS) -DJSON_TEST_STRINGS
 
+pointercheck:tests/pointercheck.cpp $(HEADERS) $(LIBFILES)
+	$(CXX) $(CXXFLAGS) -o pointercheck tests/pointercheck.cpp src/stage2_build_tape.cpp src/jsonioutil.cpp src/jsonparser.cpp src/simdjson.cpp src/stage1_find_marks.cpp  src/parsedjson.cpp src/parsedjsoniterator.cpp -I. $(LIBFLAGS)
 
-minifiercompetition: benchmark/minifiercompetition.cpp $(HEADERS) $(LIBS) $(MINIFIERHEADERS) $(LIBFILES) $(MINIFIERLIBFILES)
+minifiercompetition: benchmark/minifiercompetition.cpp $(HEADERS) submodules $(MINIFIERHEADERS) $(LIBFILES) $(MINIFIERLIBFILES)
 	$(CXX) $(CXXFLAGS) -o minifiercompetition $(LIBFILES) $(MINIFIERLIBFILES) benchmark/minifiercompetition.cpp -I. $(LIBFLAGS) $(COREDEPSINCLUDE)
 
 minify: tools/minify.cpp $(HEADERS) $(MINIFIERHEADERS) $(LIBFILES) $(MINIFIERLIBFILES)
@@ -132,28 +163,33 @@ minify: tools/minify.cpp $(HEADERS) $(MINIFIERHEADERS) $(LIBFILES) $(MINIFIERLIB
 json2json: tools/json2json.cpp $(HEADERS) $(LIBFILES)
 	$(CXX) $(CXXFLAGS) -o json2json $ tools/json2json.cpp $(LIBFILES) -I.
 
+jsonpointer: tools/jsonpointer.cpp $(HEADERS) $(LIBFILES)
+	$(CXX) $(CXXFLAGS) -o jsonpointer $ tools/jsonpointer.cpp $(LIBFILES) -I.
+
 jsonstats: tools/jsonstats.cpp $(HEADERS) $(LIBFILES)
 	$(CXX) $(CXXFLAGS) -o jsonstats $ tools/jsonstats.cpp $(LIBFILES) -I.
 
 ujdecode.o: $(UJSON4C_INCLUDE)
 	$(CC) $(CFLAGS) -c dependencies/ujson4c/src/ujdecode.c
 
-parseandstatcompetition: benchmark/parseandstatcompetition.cpp $(HEADERS) $(LIBFILES) $(LIBS)
+parseandstatcompetition: benchmark/parseandstatcompetition.cpp $(HEADERS) $(LIBFILES) submodules
 	$(CXX) $(CXXFLAGS)  -o parseandstatcompetition $(LIBFILES) benchmark/parseandstatcompetition.cpp -I. $(LIBFLAGS) $(COREDEPSINCLUDE)
 
-distinctuseridcompetition: benchmark/distinctuseridcompetition.cpp $(HEADERS) $(LIBFILES) $(LIBS)
+distinctuseridcompetition: benchmark/distinctuseridcompetition.cpp $(HEADERS) $(LIBFILES) submodules
 	$(CXX) $(CXXFLAGS)  -o distinctuseridcompetition $(LIBFILES) benchmark/distinctuseridcompetition.cpp  -I. $(LIBFLAGS) $(COREDEPSINCLUDE)
 
-parsingcompetition: benchmark/parsingcompetition.cpp $(HEADERS) $(LIBFILES) $(LIBS)
+parsingcompetition: benchmark/parsingcompetition.cpp $(HEADERS) $(LIBFILES) submodules 
+	@echo "In case of build error due to missing files, try 'make clean'"
 	$(CXX) $(CXXFLAGS)  -o parsingcompetition $(LIBFILES) benchmark/parsingcompetition.cpp -I. $(LIBFLAGS) $(COREDEPSINCLUDE)
 
-allparsingcompetition: benchmark/parsingcompetition.cpp $(HEADERS) $(LIBFILES) $(EXTRAOBJECTS) $(LIBS)
+allparsingcompetition: benchmark/parsingcompetition.cpp $(HEADERS) $(LIBFILES) $(EXTRAOBJECTS) submodules
 	$(CXX) $(CXXFLAGS)  -o allparsingcompetition $(LIBFILES) benchmark/parsingcompetition.cpp  $(EXTRAOBJECTS) -I. $(LIBFLAGS) $(COREDEPSINCLUDE) $(EXTRADEPSINCLUDE)  -DALLPARSER
 
 
-allparserscheckfile: tests/allparserscheckfile.cpp $(HEADERS) $(LIBFILES) $(EXTRAOBJECTS) $(LIBS)
+allparserscheckfile: tests/allparserscheckfile.cpp $(HEADERS) $(LIBFILES) $(EXTRAOBJECTS) submodules
 	$(CXX) $(CXXFLAGS) -o allparserscheckfile $(LIBFILES) tests/allparserscheckfile.cpp $(EXTRAOBJECTS) -I. $(LIBFLAGS) $(COREDEPSINCLUDE) $(EXTRADEPSINCLUDE)
 
+.PHONY:  clean cppcheck cleandist
 
 cppcheck:
 	cppcheck --enable=all src/*.cpp  benchmarks/*.cpp tests/*.cpp -Iinclude -I. -Ibenchmark/linux
@@ -161,7 +197,7 @@ cppcheck:
 everything: $(MAINEXECUTABLES) $(EXTRA_EXECUTABLES) $(TESTEXECUTABLES) $(COMPARISONEXECUTABLES) $(SUPPLEMENTARYEXECUTABLES)
 
 clean:
-	rm -f $(EXTRAOBJECTS) $(MAINEXECUTABLES) $(EXTRA_EXECUTABLES) $(TESTEXECUTABLES) $(COMPARISONEXECUTABLES) $(SUPPLEMENTARYEXECUTABLES)
+	rm -f submodules $(EXTRAOBJECTS) $(MAINEXECUTABLES) $(EXTRA_EXECUTABLES) $(TESTEXECUTABLES) $(COMPARISONEXECUTABLES) $(SUPPLEMENTARYEXECUTABLES)
 
 cleandist:
-	rm -f $(EXTRAOBJECTS) $(MAINEXECUTABLES) $(EXTRA_EXECUTABLES) $(TESTEXECUTABLES) $(COMPARISONEXECUTABLES) $(SUPPLEMENTARYEXECUTABLES)
+	rm -f submodules $(EXTRAOBJECTS) $(MAINEXECUTABLES) $(EXTRA_EXECUTABLES) $(TESTEXECUTABLES) $(COMPARISONEXECUTABLES) $(SUPPLEMENTARYEXECUTABLES)

Notes.md

@@ -1,4 +1,4 @@
-# Notes on simdson
+# Notes on simdjson
 
 ## Rationale:
 
@@ -21,7 +21,7 @@ may be missing the opportunity to carry out 56 operations for each branch miss.
 Many code-bases make use of SIMD and deeply pipelined, "non-branchy", processing for regular tasks. Numerical problems 
 (e.g. "matrix multiply") or simple 'bulk search' tasks (e.g. "count all the occurrences of a given character in a text", 
 "find the first occurrence of the string 'foo' in a text") frequently use this class of techniques. We are demonstrating 
-that these techniques can be applied to much more complex and less regular task.
+that these techniques can be applied to much more complex and less regular tasks.
 
 ## Design:
 
@@ -47,7 +47,7 @@ transformation to discover locations in our data that follow a structural charac
 these locations are the only place that we can expect to see the starts of the JSON 'atoms'. These locations are thus 
 treated as 'structural' ('pseudo-structural characters').
 
-This stage involves either SIMD processing over out bytes or the manipulation of bit arrays that have 1 bit corresponding 
+This stage involves either SIMD processing over bytes or the manipulation of bit arrays that have 1 bit corresponding 
 to 1 byte of input. As such, it can be quite inefficient for some inputs - it is possible to observe dozens of operations 
 taking place to discover that there are in fact no odd-numbered sequences of backslashes or quotes in a given block of 
 input. However, this inefficiency on such inputs is balanced by the fact that it costs no more to run this code over 

README.md

@@ -3,6 +3,7 @@
 [![CircleCI](https://circleci.com/gh/lemire/simdjson.svg?style=svg)](https://circleci.com/gh/lemire/simdjson)
 [![Build Status](https://img.shields.io/appveyor/ci/lemire/simdjson.svg)](https://ci.appveyor.com/project/lemire/simdjson)
 [![][license img]][license]
+[![Code Quality: Cpp](https://img.shields.io/lgtm/grade/cpp/g/lemire/simdjson.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/lemire/simdjson/context:cpp)
 
 
 ## A C++ library to see how fast we can parse JSON with complete validation.
@@ -12,10 +13,20 @@ JSON documents are everywhere on the Internet. Servers spend a lot of time parsi
 <img src="images/logo.png" width="10%">
 
 
+## Real-world usage
+
+- [Microsoft FishStore](https://github.com/microsoft/FishStore)
+- [Yandex ClickHouse](https://github.com/yandex/ClickHouse)
+
 ## Paper
 
 A description of the design and implementation of simdjson appears at https://arxiv.org/abs/1902.08318 and an informal blog post providing some background and context is at https://branchfree.org/2019/02/25/paper-parsing-gigabytes-of-json-per-second/.
 
+Some people [enjoy reading our paper](https://arxiv.org/abs/1902.08318):
+
+[<img src="images/halvarflake.png" width="50%">](https://twitter.com/halvarflake/status/1118459536686362625)
+
+
 ## Performance results
 
 simdjson uses three-quarters less instructions than state-of-the-art parser RapidJSON and fifty percent less than sajson. To our knowledge, simdjson is the first fully-validating JSON parser to run at gigabytes per second on commodity processors.
@@ -37,11 +48,15 @@ On a Skylake processor, the parsing speeds (in GB/s) of various processors on th
 | ultrajson                             | 0.42 |
 | jsmn                                  | 0.28 |
 | cJSON                                 | 0.34 |
+| JSON for Modern C++ (nlohmann/json)   | 0.10 |
 
 ## Requirements
 
 - We support platforms like Linux or macOS, as well as Windows through Visual Studio 2017 or later.
-- A processor with AVX2 (i.e., Intel processors starting with the Haswell microarchitecture released 2013 and AMD processors starting with the Zen microarchitecture released 2017).
+- A processor with 
+  - AVX2 (i.e., Intel processors starting with the Haswell microarchitecture released 2013 and AMD processors starting with the Zen microarchitecture released 2017), 
+  - or SSE 4.2 and CLMUL (i.e., Intel processors going back to Westmere released in 2010 or AMD processors starting with the Jaguar used in the PS4 and XBox One)
+  - or a 64-bit ARM processor (ARMv8-A): this covers a wide range of mobile processors, including all Apple processors currently available for sale, going back as far back as the iPhone 5s (2013).
 - A recent C++ compiler (e.g., GNU GCC or LLVM CLANG or Visual Studio 2017), we assume C++17. GNU GCC 7 or better or LLVM's clang 6 or better.
 - Some benchmark scripts assume bash and other common utilities, but they are optional.
 
@@ -51,29 +66,30 @@ This code is made available under the Apache License 2.0.
 
 Under Windows, we build some tools using the windows/dirent_portable.h file (which is outside our library code): it under the liberal (business-friendly) MIT license.
 
-## Code example
+## Code usage and example
+
+The main API involves populating a `ParsedJson` object which hosts a fully navigable document-object-model (DOM) view of the JSON document. The DOM can be accessed using [JSON Pointer](https://tools.ietf.org/html/rfc6901) paths, for example. The main function is `json_parse` which takes a string containing the JSON document as well as a reference to pre-allocated `ParsedJson` object (which can be reused multiple time). Once you have populated the `ParsedJson` object you can navigate through the DOM with an iterator (e.g., created by `ParsedJson::Iterator pjh(pj)`, see 'Navigating the parsed document').
 
 ```C
 #include "simdjson/jsonparser.h"
+using namespace simdjson;
 
 /...
 
 const char * filename = ... //
 
-// use whatever means you want to get a string of your JSON document
-std::string_view p = get_corpus(filename);
+// use whatever means you want to get a string (UTF-8) of your JSON document
+padded_string p = get_corpus(filename); 
 ParsedJson pj;
-pj.allocateCapacity(p.size()); // allocate memory for parsing up to p.size() bytes
+pj.allocate_capacity(p.size()); // allocate memory for parsing up to p.size() bytes
 const int res = json_parse(p, pj); // do the parsing, return 0 on success
 // parsing is done!
 if (res != 0) {
     // You can use the "simdjson/simdjson.h" header to access the error message
-    std::cout << "Error parsing:" << simdjson::errorMsg(res) << std::endl;
+    std::cout << "Error parsing:" << simdjson::error_message(res) << std::endl;
 }
-// You can safely delete the string content
-free((void*)p.data());
 // the ParsedJson document can be used here
-// js can be reused with other json_parse calls.
+// pj can be reused with other json_parse calls.
 ```
 
 It is also possible to use a simpler API if you do not mind having the overhead
@@ -81,18 +97,59 @@ of memory allocation with each new JSON document:
 
 ```C
 #include "simdjson/jsonparser.h"
+using namespace simdjson;
 
 /...
 
 const char * filename = ... //
-std::string_view p = get_corpus(filename);
+padded_string p = get_corpus(filename);
 ParsedJson pj = build_parsed_json(p); // do the parsing
-// you no longer need p at this point, can do aligned_free((void*)p.data())
-if( ! pj.isValid() ) {
+if( ! pj.is_valid() ) {
     // something went wrong
+    std::cout << pj.get_error_message() << std::endl;
 }
 ```
 
+Though the `padded_string` class is recommended for best performance, you can call `json_parse` and `build_parsed_json`, passing a standard `std::string` object.
+
+
+```C
+#include "simdjson/jsonparser.h"
+using namespace simdjson;
+
+/...
+std::string mystring = ... //
+ParsedJson pj;
+pj.allocate_capacity(mystring.size()); // allocate memory for parsing up to p.size() bytes
+// std::string may not overallocate so a copy will be needed
+const int res = json_parse(mystring, pj); // do the parsing, return 0 on success
+// parsing is done!
+if (res != 0) {
+    // You can use the "simdjson/simdjson.h" header to access the error message
+    std::cout << "Error parsing:" << simdjson::error_message(res) << std::endl;
+}
+// pj can be reused with other json_parse calls.
+```
+
+or
+
+```C
+#include "simdjson/jsonparser.h"
+using namespace simdjson;
+
+/...
+
+std::string mystring = ... //
+// std::string may not overallocate so a copy will be needed
+ParsedJson pj = build_parsed_json(mystring); // do the parsing
+if( ! pj.is_valid() ) {
+    // something went wrong
+    std::cout << pj.get_error_message() << std::endl;
+}
+```
+
+As needed, the `json_parse` and `build_parsed_json` functions copy the input data to a temporary buffer readable up to SIMDJSON_PADDING bytes beyond the end of the data. 
+
 ## Usage: easy single-header version
 
 See the "singleheader" repository for a single header version. See the included
@@ -103,12 +160,14 @@ copy the files in your project in your include path. You can then include them q
 #include <iostream>
 #include "simdjson.h"
 #include "simdjson.cpp"
+using namespace simdjson;
 int main(int argc, char *argv[]) {
   const char * filename = argv[1];
-  std::string_view p = get_corpus(filename);
+  padded_string p = get_corpus(filename);
   ParsedJson pj = build_parsed_json(p); // do the parsing
-  if( ! pj.isValid() ) {
+  if( ! pj.is_valid() ) {
     std::cout << "not valid" << std::endl;
+    std::cout << pj.get_error_message() << std::endl;
   } else {
     std::cout << "valid" << std::endl;
   }
@@ -116,16 +175,23 @@ int main(int argc, char *argv[]) {
 }
 ```
 
-We require hardware support for AVX2 instructions. You have to make sure that you instruct your 
-compiler to use these instructions as needed. Under compilers such as GNU GCC or LLVM clang, the
-flag `-march=native` used on a recent Intel processor (Haswell or better) is sufficient. For portability
-of the binary files you can also specify directly the Haswell processor (`-march=haswell`). You may 
-also use the flags `-mavx2 -mbmi2`. Under Visual Studio, you need to target x64 and add the 
-flag `/arch:AVX2`. 
-
 
 Note: In some settings, it might be desirable to precompile `simdjson.cpp` instead of including it.
 
+## Runtime dispatch
+
+On Intel and AMD processors, we get best performance by using the hardware support for AVX2 instructions. However, simdjson also
+runs on older Intel and AMD processors. We require a minimum feature support of SSE 4.2 and CLMUL (2010 Intel Westmere or better).
+The code automatically detects the feature set of your processor and switches to the right function at runtime (a technical
+sometimes called runtime dispatch).
+
+
+We also support 64-bit ARM. We assume NEON support, and if the cryptographic extension is available, we leverage it, at compile-time.
+There is no runtime dispatch on ARM.
+
+## Thread safety
+
+The simdjson library is single-threaded and thread safety is the responsability of the caller. If you are on an x64 processor, the runtime dispatching assigns the right code path the firs time that parsing is attempted. For safety, you should always call json_parse at least once in a single-threaded context.
 
 
 ## Usage (old-school Makefile on platforms like Linux or macOS)
@@ -208,14 +274,15 @@ make test
 
 ## Usage (CMake on Windows using Visual Studio)
 
-We assume you have a common Windows PC with at least Visual Studio 2017 and an x64 processor with AVX2 support (2013 Intel Haswell or later).
+We assume you have a common Windows PC with at least Visual Studio 2017 and an x64 processor with AVX2 support (2013 Intel Haswell or later) or SSE 4.2 + CLMUL (2010 Westmere or later). 
 
 - Grab the simdjson code from GitHub, e.g., by cloning it using [GitHub Desktop](https://desktop.github.com/).
 - Install [CMake](https://cmake.org/download/). When you install it, make sure to ask that `cmake` be made available from the command line. Please choose a recent version of cmake.
 - Create a subdirectory within simdjson, such as `VisualStudio`.
 - Using a shell, go to this newly created directory.
-- Type `cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..` in the shell while in the `VisualStudio` repository. (Alternatively, if you want to build a DLL, you may use the command line `cmake -DCMAKE_GENERATOR_PLATFORM=x64 -DSIMDJSON_BUILD_STATIC=OFF ..`.)
-- This last command created a Visual Studio solution file in the newly created directory (e.g., `simdjson.sln`). Open this file in Visual Studio. You should now be able to build the project and run the tests. For example, in the `Solution Explorer` window (available from the `View` menu), right-click `ALL_BUILD` and select `Build`. To test the code, still in the `Solution Explorer` window, select `RUN_TESTS` and select `Build`.
+- Type `cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..` in the shell while in the `VisualStudio` repository. (Alternatively, if you want to build a DLL, you may use the command line `cmake -DCMAKE_GENERATOR_PLATFORM=x64 -DSIMDJSON_BUILD_STATIC=OFF ..`.) 
+- This last command (`cmake ...`) created a Visual Studio solution file in the newly created directory (e.g., `simdjson.sln`). Open this file in Visual Studio. You should now be able to build the project and run the tests. For example, in the `Solution Explorer` window (available from the `View` menu), right-click `ALL_BUILD` and select `Build`. To test the code, still in the `Solution Explorer` window, select `RUN_TESTS` and select `Build`.
+
 
 
 ## Usage (Using `vcpkg` on Windows, Linux and MacOS)
@@ -254,18 +321,17 @@ If you find the version of `simdjson` shipped with `vcpkg` is out-of-date, feel
 - `json2json mydoc.json` parses the document, constructs a model and then dumps back the result to standard output.
 - `json2json -d mydoc.json` parses the document, constructs a model and then dumps model (as a tape) to standard output. The tape format is described in the accompanying file `tape.md`.
 - `minify mydoc.json` minifies the JSON document, outputting the result to standard output. Minifying means to remove the unneeded white space characters.
+- `jsonpointer mydoc.json <jsonpath> <jsonpath> ... <jsonpath>` parses the document, constructs a model and then processes a series of [JSON Pointer paths](https://tools.ietf.org/html/rfc6901). The result is itself a JSON document.
 
 ## Scope
 
-We provide a fast parser. It fully validates the input according to the various specifications.
+We provide a fast parser, that fully validates an input according to various specifications.
 The parser builds a useful immutable (read-only) DOM (document-object model) which can be later accessed.
 
 To simplify the engineering, we make some assumptions.
 
-- We support UTF-8 (and thus ASCII), nothing else (no Latin, no UTF-16). We do not believe that this is a genuine limitation in the sense that we do not think that there is any serious application that needs to process JSON data without an ASCII or UTF-8 encoding.
-- All strings in the JSON document may have up to 4294967295 bytes in UTF-8 (4GB). To enforce this constraint, we refuse to parse a document that contains more than 4294967295 bytes (4GB). This should accomodate most JSON documents.
-- We assume AVX2 support which is available in all recent mainstream x86 processors produced by AMD and Intel. No support for non-x86 processors is included though it can be done. We plan to support ARM processors (help is invited).
-- In cases of failure, we just report a failure without any indication as to the nature of the problem. (This can be easily improved without affecting performance.)
+- We support UTF-8 (and thus ASCII), nothing else (no Latin, no UTF-16). We do not believe this is a genuine limitation, because we do not think there is any serious application that needs to process JSON data without an ASCII or UTF-8 encoding. If the UTF-8 contains a leading BOM, it should be omitted: the user is responsible for detecting and skipping the BOM; UTF-8 BOMs are discouraged.
+- All strings in the JSON document may have up to 4294967295 bytes in UTF-8 (4GB). To enforce this constraint, we refuse to parse a document that contains more than 4294967295 bytes (4GB). This should accommodate most JSON documents.
 - As allowed by the specification, we allow repeated keys within an object (other parsers like sajson do the same).
 - Performance is optimized for JSON documents spanning at least a tens kilobytes up to many megabytes: the performance issues with having to parse many tiny JSON documents or one truly enormous JSON document are different.
 
@@ -275,9 +341,13 @@ _We do not aim to provide a general-purpose JSON library._ A library like RapidJ
 
 - The input string is unmodified. (Parsers like sajson and RapidJSON use the input string as a buffer.)
 - We parse integers and floating-point numbers as separate types which allows us to support large 64-bit integers in [-9223372036854775808,9223372036854775808), like a Java `long` or a C/C++ `long long`. Among the parsers that differentiate between integers and floating-point numbers, not all support 64-bit integers. (For example, sajson rejects JSON files with integers larger than or equal to 2147483648. RapidJSON will parse a file containing an overly long integer like 18446744073709551616 as a floating-point number.) When we cannot represent exactly an integer as a signed 64-bit value, we reject the JSON document.
-- We do full UTF-8 validation as part of the parsing. (Parsers like fastjson, gason and dropbox json11 do not do UTF-8 validation.)
+- We support the full range of 64-bit floating-point numbers (binary64). The values range from ` std::numeric_limits<double>::lowest()`  to `std::numeric_limits<double>::max()`, so from -1.7976e308 all the way to 1.7975e308. Extreme values (less or equal to -1e308, greater or equal to 1e308) are rejected: we refuse to parse the input document.
+- We test for accurate float parsing with a bound on the [unit of least precision (ULP)](https://en.wikipedia.org/wiki/Unit_in_the_last_place) of one. Practically speaking, this implies 15 digits of accuracy or better.
+- We do full UTF-8 validation as part of the parsing. (Parsers like fastjson, gason and dropbox json11 do not do UTF-8 validation. The sajson parser does incomplete UTF-8 validation, accepting code point
+sequences like 0xb1 0x87.)
 - We fully validate the numbers. (Parsers like gason and ultranjson will accept `[0e+]` as valid JSON.)
 - We validate string content for unescaped characters. (Parsers like fastjson and ultrajson accept unescaped line breaks and tabs in strings.)
+- We fully validate the white-space characters outside of the strings. Parsers like RapidJSON will accept JSON documents with null characters outside of strings.
 
 ## Architecture
 
@@ -286,13 +356,27 @@ The parser works in two stages:
 - Stage 1. (Find marks) Identifies quickly structure elements, strings, and so forth. We validate UTF-8 encoding at that stage.
 - Stage 2. (Structure building) Involves constructing a "tree" of sort (materialized as a tape) to navigate through the data. Strings and numbers are parsed at this stage.
 
+## JSON Pointer
+
+We can navigate the parsed JSON using JSON Pointers as per the [RFC6901 standard](https://tools.ietf.org/html/rfc6901).
+
+You can build a tool (jsonpointer) to parse a JSON document and then issue an array of JSON Pointer queries:
+
+```
+make jsonpointer 
+ ./jsonpointer jsonexamples/small/demo.json /Image/Width /Image/Height /Image/IDs/2
+ ./jsonpointer jsonexamples/twitter.json /statuses/0/id /statuses/1/id /statuses/2/id /statuses/3/id /statuses/4/id /statuses/5/id
+```
+
+In C++, given a `ParsedJson`, we can move to a node with the `move_to` method, passing a `std::string` representing the JSON Pointer query.
+
 ## Navigating the parsed document
 
 Here is a code sample to dump back the parsed JSON to a string:
 
 ```c
-    ParsedJson::iterator pjh(pj);
-    if (!pjh.isOk()) {
+    ParsedJson::Iterator pjh(pj);
+    if (!pjh.is_ok()) {
       std::cerr << " Could not iterate parsed result. " << std::endl;
       return EXIT_FAILURE;
     }
@@ -300,7 +384,7 @@ Here is a code sample to dump back the parsed JSON to a string:
     //
     // where compute_dump is :
 
-void compute_dump(ParsedJson::iterator &pjh) {
+void compute_dump(ParsedJson::Iterator &pjh) {
   if (pjh.is_object()) {
     std::cout << "{";
     if (pjh.down()) {
@@ -338,46 +422,21 @@ void compute_dump(ParsedJson::iterator &pjh) {
 The following function will find all user.id integers:
 
 ```C
-void simdjson_traverse(std::vector<int64_t> &answer, ParsedJson::iterator &i) {
-  switch (i.get_type()) {
-  case '{':
-    if (i.down()) {
-      do {
-        bool founduser = equals(i.get_string(), "user");
-        i.next(); // move to value
-        if (i.is_object()) {
-          if (founduser && i.move_to_key("id")) {
+void simdjson_scan(std::vector<int64_t> &answer, ParsedJson::Iterator &i) {
+   while(i.move_forward()) {
+     if(i.get_scope_type() == '{') {
+       bool found_user = (i.get_string_length() == 4) && (memcmp(i.get_string(), "user", 4) == 0);
+       i.move_to_value();
+       if(found_user) {
+          if(i.is_object() && i.move_to_key("id",2)) {
             if (i.is_integer()) {
               answer.push_back(i.get_integer());
             }
             i.up();
           }
-          simdjson_traverse(answer, i);
-        } else if (i.is_array()) {
-          simdjson_traverse(answer, i);
-        }
-      } while (i.next());
-      i.up();
-    }
-    break;
-  case '[':
-    if (i.down()) {
-      do {
-        if (i.is_object_or_array()) {
-          simdjson_traverse(answer, i);
-        }
-      } while (i.next());
-      i.up();
-    }
-    break;
-  case 'l':
-  case 'd':
-  case 'n':
-  case 't':
-  case 'f':
-  default:
-    break;
-  }
+       }
+     }
+   }
 }
 ```
 
@@ -404,11 +463,31 @@ make allparsingcompetition
 ./allparsingcompetition myfile.json
 ```
 
+Both the `parsingcompetition` and `allparsingcompetition` tools take a `-t` flag which produces
+a table-oriented output that can be conventiently parsed by other tools.
+
+
+## Docker
+
+One can run tests and benchmarks using docker. It especially makes sense under Linux. A privileged access may be needed to get performance counters.
+
+```
+git clone https://github.com/lemire/simdjson.git
+cd simdjson
+docker build -t simdjson . 
+docker run --privileged -t simdjson
+```
+
 ## Other programming languages
 
+We distinguish between "bindings" (which just wrap the C++ code) and a port to another programming language (which reimplements everything).
+
 - [pysimdjson](https://github.com/TkTech/pysimdjson): Python bindings for the simdjson project.
-- [SimdJsonSharp](https://github.com/EgorBo/SimdJsonSharp): C# version for .NET Core
+- [simdjson-rs](https://github.com/Licenser/simdjson-rs): Rust port
+- [simdjson-rust](https://github.com/SunDoge/simdjson-rust): Rust wrapper (bindings)
+- [SimdJsonSharp](https://github.com/EgorBo/SimdJsonSharp): C# version for .NET Core (bindings and full port)
 - [simdjson_nodejs](https://github.com/luizperes/simdjson_nodejs): Node.js bindings for the simdjson project.
+- [simdjson_php](https://github.com/crazyxman/simdjson_php): PHP bindings for the simdjson project.
 
 ## Various References
 
@@ -460,11 +539,11 @@ A character is pseudo-structural if and only if:
 
 1. Not enclosed in quotes, AND
 2. Is a non-whitespace character, AND
-3. It's preceding character is either:
+3. Its preceding character is either:
    (a) a structural character, OR
    (b) whitespace.
 
-This helps as we redefine some new characters as pseudo-structural such as the characters 1, 1, G, n in the following:
+This helps as we redefine some new characters as pseudo-structural such as the characters 1, G, n in the following:
 
 > { "foo" : 1.5, "bar" : 1.5 GEOFF_IS_A_DUMMY bla bla , "baz", null }
 
@@ -492,5 +571,11 @@ This helps as we redefine some new characters as pseudo-structural such as the c
 - Yang, Shiyang. Validation of XML Document Based on Parallel Bit Stream Technology. Diss. Applied Sciences: School of Computing Science, 2013.
 - N. Nakasato, "Implementation of a parallel tree method on a GPU", Journal of Computational Science, vol. 3, no. 3, pp. 132-141, 2012.
 
+
+## Funding
+
+The work is supported by the Natural Sciences and Engineering Research Council of Canada under grant number RGPIN-2017-03910. 
+
+
 [license]: LICENSE
 [license img]: https://img.shields.io/badge/License-Apache%202-blue.svg

amalgamation.sh

@@ -13,6 +13,7 @@ AMAL_C="simdjson.cpp"
 
 # order does not matter
 ALLCFILES="
+$SCRIPTPATH/src/simdjson.cpp
 $SCRIPTPATH/src/jsonioutil.cpp
 $SCRIPTPATH/src/jsonminifier.cpp
 $SCRIPTPATH/src/jsonparser.cpp
@@ -25,18 +26,32 @@ $SCRIPTPATH/src/parsedjsoniterator.cpp
 # order matters
 ALLCHEADERS="
 $SCRIPTPATH/include/simdjson/simdjson_version.h
-$SCRIPTPATH/include/simdjson/simdjson.h
 $SCRIPTPATH/include/simdjson/portability.h
+$SCRIPTPATH/include/simdjson/isadetection.h
+$SCRIPTPATH/include/simdjson/simdjson.h
 $SCRIPTPATH/include/simdjson/common_defs.h
+$SCRIPTPATH/include/simdjson/padded_string.h
 $SCRIPTPATH/include/simdjson/jsoncharutils.h
 $SCRIPTPATH/include/simdjson/jsonformatutils.h
 $SCRIPTPATH/include/simdjson/jsonioutil.h
 $SCRIPTPATH/include/simdjson/simdprune_tables.h
-$SCRIPTPATH/include/simdjson/simdutf8check.h
+$SCRIPTPATH/include/simdjson/simdutf8check_haswell.h
+$SCRIPTPATH/include/simdjson/simdutf8check_westmere.h
+$SCRIPTPATH/include/simdjson/simdutf8check_arm64.h
 $SCRIPTPATH/include/simdjson/jsonminifier.h
 $SCRIPTPATH/include/simdjson/parsedjson.h
 $SCRIPTPATH/include/simdjson/stage1_find_marks.h
+$SCRIPTPATH/include/simdjson/stage1_find_marks_flatten.h
+$SCRIPTPATH/include/simdjson/stage1_find_marks_flatten_haswell.h
+$SCRIPTPATH/include/simdjson/stage1_find_marks_macros.h
+$SCRIPTPATH/include/simdjson/stage1_find_marks_westmere.h
+$SCRIPTPATH/include/simdjson/stage1_find_marks_haswell.h
+$SCRIPTPATH/include/simdjson/stage1_find_marks_arm64.h
 $SCRIPTPATH/include/simdjson/stringparsing.h
+$SCRIPTPATH/include/simdjson/stringparsing_macros.h
+$SCRIPTPATH/include/simdjson/stringparsing_westmere.h
+$SCRIPTPATH/include/simdjson/stringparsing_haswell.h
+$SCRIPTPATH/include/simdjson/stringparsing_arm64.h
 $SCRIPTPATH/include/simdjson/numberparsing.h
 $SCRIPTPATH/include/simdjson/stage2_build_tape.h
 $SCRIPTPATH/include/simdjson/jsonparser.h
@@ -99,10 +114,13 @@ cat <<< '
 #include "simdjson.h"
 #include "simdjson.cpp"
 int main(int argc, char *argv[]) {
+  if(argc < 2) {
+    std::cerr << "Please specify a filename " << std::endl;
+  }
   const char * filename = argv[1];
-  std::string_view p = get_corpus(filename);
-  ParsedJson pj = build_parsed_json(p); // do the parsing
-  if( ! pj.isValid() ) {
+  simdjson::padded_string p = simdjson::get_corpus(filename);
+  simdjson::ParsedJson pj = simdjson::build_parsed_json(p); // do the parsing
+  if( ! pj.is_valid() ) {
     std::cout << "not valid" << std::endl;
   } else {
     std::cout << "valid" << std::endl;
@@ -122,16 +140,16 @@ echo "Giving final instructions:"
 CPPBIN=${DEMOCPP%%.*}
 
 echo "Try :"
-echo "c++ -march=native -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json "
+echo "c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json "
 
 SINGLEHDR=$SCRIPTPATH/singleheader
 echo "Copying files to $SCRIPTPATH/singleheader "
 mkdir -p $SINGLEHDR
-echo "c++ -march=native -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json " > $SINGLEHDR/README.md
+echo "c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json " > $SINGLEHDR/README.md
 cp ${AMAL_C} ${AMAL_H}  ${DEMOCPP} $SINGLEHDR
 ls $SINGLEHDR
 
-cd $SINGLEHDR && c++ -march=native -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json
+cd $SINGLEHDR && c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json
 
 lowercase(){
     echo "$1" | tr 'A-Z' 'a-z'

benchmark/benchmark.h

@@ -18,7 +18,7 @@ const char *unitname = "cycles";
                    :                                                           \
                    :                              /* no read only */           \
                    "%rax", "%rbx", "%rcx", "%rdx" /* clobbers */               \
-                   );                                                          \
+    );                                                                         \
     (cycles) = ((uint64_t)cyc_high << 32) | cyc_low;                           \
   } while (0)
 
@@ -32,7 +32,7 @@ const char *unitname = "cycles";
                    : "=r"(cyc_high), "=r"(cyc_low)                             \
                    : /* no read only registers */                              \
                    : "%rax", "%rbx", "%rcx", "%rdx" /* clobbers */             \
-                   );                                                          \
+    );                                                                         \
     (cycles) = ((uint64_t)cyc_high << 32) | cyc_low;                           \
   } while (0)
 
@@ -92,7 +92,7 @@ double diff(timespec start, timespec end) {
     if (verbose)                                                               \
       printf("%-40s\t: ", name);                                               \
     else                                                                       \
-      printf("\"%s\"\t", name);                                                \
+      printf("\"%-40s\"", name);                                               \
     fflush(NULL);                                                              \
     uint64_t cycles_start, cycles_final, cycles_diff;                          \
     uint64_t min_diff = (uint64_t)-1;                                          \
@@ -106,7 +106,7 @@ double diff(timespec start, timespec end) {
       clock_gettime(CLOCK_REALTIME, &time1);                                   \
       RDTSC_START(cycles_start);                                               \
       if (test != expected) {                                                  \
-        fprintf(stderr, "not expected (%d , %d )", (int)test, (int)expected);           \
+        fprintf(stderr, "not expected (%d , %d )", (int)test, (int)expected);  \
         break;                                                                 \
       }                                                                        \
       RDTSC_STOP(cycles_final);                                                \
@@ -128,11 +128,14 @@ double diff(timespec start, timespec end) {
     double max_gb_per_s =                                                      \
         ((double)S) / ((min_sumclockdiff)*1000.0 * 1000.0 * 1000.0);           \
     if (verbose)                                                               \
-      printf(" %.3f %s per input byte (best) ", cycle_per_op, unitname);       \
+      printf(" %7.3f %s per input byte (best) ", cycle_per_op, unitname);      \
     if (verbose)                                                               \
-      printf(" %.3f %s per input byte (avg) ", avg_cycle_per_op, unitname);    \
+      printf(" %7.3f %s per input byte (avg) ", avg_cycle_per_op, unitname);   \
+    if (verbose)                                                               \
+      printf(" %7.3f GB/s (error margin: %.3f GB/s)", max_gb_per_s,            \
+             -avg_gb_per_s + max_gb_per_s);                                    \
     if (!verbose)                                                              \
-      printf(" %.3f %.3f %.3f %.3f ", cycle_per_op,                            \
+      printf(" %20.3f %20.3f %20.3f %20.3f ", cycle_per_op,                    \
              avg_cycle_per_op - cycle_per_op, max_gb_per_s,                    \
              -avg_gb_per_s + max_gb_per_s);                                    \
     printf("\n");                                                              \

benchmark/distinctuseridcompetition.cpp

@@ -14,78 +14,70 @@
 #include "sajson.h"
 
 using namespace rapidjson;
-using namespace std;
 
 bool equals(const char *s1, const char *s2) { return strcmp(s1, s2) == 0; }
 
-void remove_duplicates(vector<int64_t> &v) {
+void remove_duplicates(std::vector<int64_t> &v) {
   std::sort(v.begin(), v.end());
   auto last = std::unique(v.begin(), v.end());
   v.erase(last, v.end());
 }
 
-void print_vec(vector<int64_t> &v) {
+void print_vec(const std::vector<int64_t> &v) {
   for (auto i : v) {
     std::cout << i << " ";
   }
   std::cout << std::endl;
 }
 
-void simdjson_traverse(std::vector<int64_t> &answer, ParsedJson::iterator &i) {
-  switch (i.get_type()) {
-  case '{':
-    if (i.down()) {
-      do {
-        bool founduser = equals(i.get_string(), "user");
-        i.next(); // move to value
-        if (i.is_object()) {
-          if (founduser && i.move_to_key("id")) {
-            if (i.is_integer()) {
-              answer.push_back(i.get_integer());
-            }
-            i.up();
+void simdjson_scan(std::vector<int64_t> &answer,
+                   simdjson::ParsedJson::Iterator &i) {
+  while (i.move_forward()) {
+    if (i.get_scope_type() == '{') {
+      bool found_user = (i.get_string_length() == 4) &&
+                        (memcmp(i.get_string(), "user", 4) == 0);
+      i.move_to_value();
+      if (found_user) {
+        if (i.is_object() && i.move_to_key("id", 2)) {
+          if (i.is_integer()) {
+            answer.push_back(i.get_integer());
           }
-          simdjson_traverse(answer, i);
-        } else if (i.is_array()) {
-          simdjson_traverse(answer, i);
+          i.up();
         }
-      } while (i.next());
-      i.up();
+      }
     }
-    break;
-  case '[':
-    if (i.down()) {
-      do {
-        if (i.is_object_or_array()) {
-          simdjson_traverse(answer, i);
-        }
-      } while (i.next());
-      i.up();
-    }
-    break;
-  case 'l':
-  case 'd':
-  case 'n':
-  case 't':
-  case 'f':
-  default:
-    break;
   }
 }
 
-std::vector<int64_t> simdjson_computestats(const std::string_view &p) {
+__attribute__((noinline)) std::vector<int64_t>
+simdjson_just_dom(simdjson::ParsedJson &pj) {
   std::vector<int64_t> answer;
-  ParsedJson pj = build_parsed_json(p);
-  if (!pj.isValid()) {
+  simdjson::ParsedJson::Iterator i(pj);
+  simdjson_scan(answer, i);
+  remove_duplicates(answer);
+  return answer;
+}
+
+__attribute__((noinline)) std::vector<int64_t>
+simdjson_compute_stats(const simdjson::padded_string &p) {
+  std::vector<int64_t> answer;
+  simdjson::ParsedJson pj = simdjson::build_parsed_json(p);
+  if (!pj.is_valid()) {
     return answer;
   }
-  ParsedJson::iterator i(pj);
-
-  simdjson_traverse(answer, i);
+  simdjson::ParsedJson::Iterator i(pj);
+  simdjson_scan(answer, i);
   remove_duplicates(answer);
   return answer;
 }
 
+__attribute__((noinline)) bool
+simdjson_just_parse(const simdjson::padded_string &p) {
+  simdjson::ParsedJson pj = simdjson::build_parsed_json(p);
+  bool answer = !pj.is_valid();
+  return answer;
+}
+
 void sajson_traverse(std::vector<int64_t> &answer, const sajson::value &node) {
   using namespace sajson;
   switch (node.get_type()) {
@@ -98,22 +90,26 @@ void sajson_traverse(std::vector<int64_t> &answer, const sajson::value &node) {
   }
   case TYPE_OBJECT: {
     auto length = node.get_length();
+    // sajson has O(log n) find_object_key, but we still visit each node anyhow
+    // because we need to visit all values.
     for (auto i = 0u; i < length; ++i) {
-      if (equals(node.get_object_key(i).data(), "user")) { // found a user!!!
-        auto uservalue = node.get_object_value(i);         // get the value
-        if (uservalue.get_type() ==
+      auto key = node.get_object_key(i); // expected: sajson::string
+      bool found_user =
+          (key.length() == 4) && (memcmp(key.data(), "user", 4) == 0);
+      if (found_user) {                             // found a user!!!
+        auto user_value = node.get_object_value(i); // get the value
+        if (user_value.get_type() ==
             TYPE_OBJECT) { // the value should be an object
-          auto uservaluelength = uservalue.get_length();
-          for (auto j = 0u; j < uservaluelength;
-               ++j) { // go through the children
-            if (equals(uservalue.get_object_key(j).data(),
-                       "id")) { // ah ah found id
-              auto v = uservalue.get_object_value(j);
-              if (v.get_type() == TYPE_INTEGER) { // check that it is an integer
-                answer.push_back(v.get_integer_value()); // record it!
-              } else if (v.get_type() == TYPE_DOUBLE) {
-                answer.push_back((int64_t)v.get_double_value()); // record it!
-              }
+          // now we know that we only need one value
+          auto user_value_length = user_value.get_length();
+          auto right_index =
+              user_value.find_object_key(sajson::string("id", 2));
+          if (right_index < user_value_length) {
+            auto v = user_value.get_object_value(right_index);
+            if (v.get_type() == TYPE_INTEGER) { // check that it is an integer
+              answer.push_back(v.get_integer_value()); // record it!
+            } else if (v.get_type() == TYPE_DOUBLE) {
+              answer.push_back((int64_t)v.get_double_value()); // record it!
             }
           }
         }
@@ -134,13 +130,23 @@ void sajson_traverse(std::vector<int64_t> &answer, const sajson::value &node) {
   }
 }
 
-std::vector<int64_t> sasjon_computestats(const std::string_view &p) {
+__attribute__((noinline)) std::vector<int64_t>
+sasjon_just_dom(sajson::document &d) {
+  std::vector<int64_t> answer;
+  sajson_traverse(answer, d.get_root());
+  remove_duplicates(answer);
+  return answer;
+}
+
+__attribute__((noinline)) std::vector<int64_t>
+sasjon_compute_stats(const simdjson::padded_string &p) {
   std::vector<int64_t> answer;
   char *buffer = (char *)malloc(p.size());
   memcpy(buffer, p.data(), p.size());
   auto d = sajson::parse(sajson::dynamic_allocation(),
                          sajson::mutable_string_view(p.size(), buffer));
   if (!d.is_valid()) {
+    free(buffer);
     return answer;
   }
   sajson_traverse(answer, d.get_root());
@@ -149,12 +155,25 @@ std::vector<int64_t> sasjon_computestats(const std::string_view &p) {
   return answer;
 }
 
+__attribute__((noinline)) bool
+sasjon_just_parse(const simdjson::padded_string &p) {
+  char *buffer = (char *)malloc(p.size());
+  memcpy(buffer, p.data(), p.size());
+  auto d = sajson::parse(sajson::dynamic_allocation(),
+                         sajson::mutable_string_view(p.size(), buffer));
+  bool answer = !d.is_valid();
+  free(buffer);
+  return answer;
+}
+
 void rapid_traverse(std::vector<int64_t> &answer, const rapidjson::Value &v) {
   switch (v.GetType()) {
   case kObjectType:
     for (Value::ConstMemberIterator m = v.MemberBegin(); m != v.MemberEnd();
          ++m) {
-      if (equals(m->name.GetString(), "user")) {
+      bool found_user = (m->name.GetStringLength() == 4) &&
+                        (memcmp(m->name.GetString(), "user", 4) == 0);
+      if (found_user) {
         const rapidjson::Value &child = m->value;
         if (child.GetType() == kObjectType) {
           for (Value::ConstMemberIterator k = child.MemberBegin();
@@ -187,7 +206,16 @@ void rapid_traverse(std::vector<int64_t> &answer, const rapidjson::Value &v) {
   }
 }
 
-std::vector<int64_t> rapid_computestats(const std::string_view &p) {
+__attribute__((noinline)) std::vector<int64_t>
+rapid_just_dom(rapidjson::Document &d) {
+  std::vector<int64_t> answer;
+  rapid_traverse(answer, d);
+  remove_duplicates(answer);
+  return answer;
+}
+
+__attribute__((noinline)) std::vector<int64_t>
+rapid_compute_stats(const simdjson::padded_string &p) {
   std::vector<int64_t> answer;
   char *buffer = (char *)malloc(p.size() + 1);
   memcpy(buffer, p.data(), p.size());
@@ -195,6 +223,7 @@ std::vector<int64_t> rapid_computestats(const std::string_view &p) {
   rapidjson::Document d;
   d.ParseInsitu<kParseValidateEncodingFlag>(buffer);
   if (d.HasParseError()) {
+    free(buffer);
     return answer;
   }
   rapid_traverse(answer, d);
@@ -203,15 +232,27 @@ std::vector<int64_t> rapid_computestats(const std::string_view &p) {
   return answer;
 }
 
+__attribute__((noinline)) bool
+rapid_just_parse(const simdjson::padded_string &p) {
+  char *buffer = (char *)malloc(p.size() + 1);
+  memcpy(buffer, p.data(), p.size());
+  buffer[p.size()] = '\0';
+  rapidjson::Document d;
+  d.ParseInsitu<kParseValidateEncodingFlag>(buffer);
+  bool answer = d.HasParseError();
+  free(buffer);
+  return answer;
+}
+
 int main(int argc, char *argv[]) {
   bool verbose = false;
-  bool justdata = false;
+  bool just_data = false;
 
   int c;
   while ((c = getopt(argc, argv, "vt")) != -1)
     switch (c) {
     case 't':
-      justdata = true;
+      just_data = true;
       break;
     case 'v':
       verbose = true;
@@ -220,19 +261,22 @@ int main(int argc, char *argv[]) {
       abort();
     }
   if (optind >= argc) {
-    cerr << "Using different parsers, we compute the content statistics of "
-            "JSON documents.\n";
-    cerr << "Usage: " << argv[0] << " <jsonfile>\n";
-    cerr << "Or " << argv[0] << " -v <jsonfile>\n";
+    std::cerr
+        << "Using different parsers, we compute the content statistics of "
+           "JSON documents."
+        << std::endl;
+    std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
+    std::cerr << "Or " << argv[0] << " -v <jsonfile>" << std::endl;
     exit(1);
   }
   const char *filename = argv[optind];
   if (optind + 1 < argc) {
-    cerr << "warning: ignoring everything after " << argv[optind + 1] << endl;
+    std::cerr << "warning: ignoring everything after " << argv[optind + 1]
+              << std::endl;
   }
-  std::string_view p;
+  simdjson::padded_string p;
   try {
-    p = get_corpus(filename);
+    simdjson::get_corpus(filename).swap(p);
   } catch (const std::exception &e) { // caught by reference to base
     std::cout << "Could not load the file " << filename << std::endl;
     return EXIT_FAILURE;
@@ -248,17 +292,17 @@ int main(int argc, char *argv[]) {
       std::cout << p.size() << " B ";
     std::cout << std::endl;
   }
-  std::vector<int64_t> s1 = simdjson_computestats(p);
+  std::vector<int64_t> s1 = simdjson_compute_stats(p);
   if (verbose) {
     printf("simdjson: ");
     print_vec(s1);
   }
-  std::vector<int64_t> s2 = rapid_computestats(p);
+  std::vector<int64_t> s2 = rapid_compute_stats(p);
   if (verbose) {
     printf("rapid:    ");
     print_vec(s2);
   }
-  std::vector<int64_t> s3 = sasjon_computestats(p);
+  std::vector<int64_t> s3 = sasjon_compute_stats(p);
   if (verbose) {
     printf("sasjon:   ");
     print_vec(s3);
@@ -267,17 +311,37 @@ int main(int argc, char *argv[]) {
   assert(s1 == s3);
   size_t size = s1.size();
 
-  int repeat = 50;
+  int repeat = 500;
   int volume = p.size();
-  if(justdata) {
-    printf("name cycles_per_byte cycles_per_byte_err  gb_per_s gb_per_s_err \n");
+  if (just_data) {
+    printf(
+        "name cycles_per_byte cycles_per_byte_err  gb_per_s gb_per_s_err \n");
   }
-  BEST_TIME("simdjson  ", simdjson_computestats(p).size(), size, , repeat,
-            volume, !justdata);
-
-  BEST_TIME("rapid  ", rapid_computestats(p).size(), size, , repeat, volume,
-            !justdata);
-  BEST_TIME("sasjon  ", sasjon_computestats(p).size(), size, , repeat, volume,
-            !justdata);
-  aligned_free((void*)p.data());
+  BEST_TIME("simdjson  ", simdjson_compute_stats(p).size(), size, , repeat,
+            volume, !just_data);
+  BEST_TIME("rapid  ", rapid_compute_stats(p).size(), size, , repeat, volume,
+            !just_data);
+  BEST_TIME("sasjon  ", sasjon_compute_stats(p).size(), size, , repeat, volume,
+            !just_data);
+  BEST_TIME("simdjson (just parse)  ", simdjson_just_parse(p), false, , repeat,
+            volume, !just_data);
+  BEST_TIME("rapid  (just parse) ", rapid_just_parse(p), false, , repeat,
+            volume, !just_data);
+  BEST_TIME("sasjon (just parse) ", sasjon_just_parse(p), false, , repeat,
+            volume, !just_data);
+  simdjson::ParsedJson dsimdjson = simdjson::build_parsed_json(p);
+  BEST_TIME("simdjson (just dom)  ", simdjson_just_dom(dsimdjson).size(), size,
+            , repeat, volume, !just_data);
+  char *buffer = (char *)malloc(p.size());
+  memcpy(buffer, p.data(), p.size());
+  rapidjson::Document drapid;
+  drapid.ParseInsitu<kParseValidateEncodingFlag>(buffer);
+  BEST_TIME("rapid  (just dom) ", rapid_just_dom(drapid).size(), size, , repeat,
+            volume, !just_data);
+  memcpy(buffer, p.data(), p.size());
+  auto dsasjon = sajson::parse(sajson::dynamic_allocation(),
+                               sajson::mutable_string_view(p.size(), buffer));
+  BEST_TIME("sasjon (just dom) ", sasjon_just_dom(dsasjon).size(), size, ,
+            repeat, volume, !just_data);
+  free(buffer);
 }

benchmark/linux/linux-perf-events.h

@@ -11,16 +11,17 @@
 #include <cstring> // for memset
 #include <stdexcept>
 
+#include <iostream>
 #include <vector>
 
-
 template <int TYPE = PERF_TYPE_HARDWARE> class LinuxEvents {
   int fd;
-  bool working; 
+  bool working;
   perf_event_attr attribs;
   int num_events;
   std::vector<uint64_t> temp_result_vec;
   std::vector<uint64_t> ids;
+
 public:
   explicit LinuxEvents(std::vector<int> config_vec) : fd(0), working(true) {
     memset(&attribs, 0, sizeof(attribs));
@@ -38,6 +39,7 @@ public:
 
     int group = -1; // no group
     num_events = config_vec.size();
+    ids.resize(config_vec.size());
     uint32_t i = 0;
     for (auto config : config_vec) {
       attribs.config = config;
@@ -71,7 +73,7 @@ public:
       report_error("ioctl(PERF_EVENT_IOC_DISABLE)");
     }
 
-    if (read(fd, &temp_result_vec[0], temp_result_vec.size() * 8) == -1) {
+    if (read(fd, temp_result_vec.data(), temp_result_vec.size() * 8) == -1) {
       report_error("read");
     }
     // our actual results are in slots 1,3,5, ... of this structure
@@ -83,8 +85,9 @@ public:
 
 private:
   void report_error(const std::string &context) {
-     if(working) std::cerr << (context + ": " + std::string(strerror(errno))) << std::endl;
-     working = false;
+    if (working)
+      std::cerr << (context + ": " + std::string(strerror(errno))) << std::endl;
+    working = false;
   }
 };
 #endif

benchmark/minifiercompetition.cpp

@@ -1,5 +1,5 @@
-#include <unistd.h>
 #include <iostream>
+#include <unistd.h>
 
 #include "benchmark.h"
 #include "simdjson/jsonioutil.h"
@@ -14,11 +14,10 @@
 #include "rapidjson/writer.h"
 #include "sajson.h"
 
-
+using namespace simdjson;
 using namespace rapidjson;
-using namespace std;
 
-std::string rapidstringmeInsitu(char *json) {
+std::string rapid_stringme_insitu(char *json) {
   Document d;
   d.ParseInsitu(json);
   if (d.HasParseError()) {
@@ -31,7 +30,7 @@ std::string rapidstringmeInsitu(char *json) {
   return buffer.GetString();
 }
 
-std::string rapidstringme(char *json) {
+std::string rapid_stringme(char *json) {
   Document d;
   d.Parse(json);
   if (d.HasParseError()) {
@@ -47,29 +46,28 @@ std::string rapidstringme(char *json) {
 int main(int argc, char *argv[]) {
   int c;
   bool verbose = false;
-  bool justdata = false;
+  bool just_data = false;
 
-  while ((c = getopt (argc, argv, "vt")) != -1)
-    switch (c)
-      {
-      case 't':
-        justdata = true;
-        break;
-      case 'v':
-        verbose = true;
-        break;
-      default:
-        abort ();
-      }
+  while ((c = getopt(argc, argv, "vt")) != -1)
+    switch (c) {
+    case 't':
+      just_data = true;
+      break;
+    case 'v':
+      verbose = true;
+      break;
+    default:
+      abort();
+    }
   if (optind >= argc) {
-    cerr << "Usage: " << argv[0] << " <jsonfile>" << endl;
+    std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
     exit(1);
   }
-  const char * filename = argv[optind];
-  std::string_view p;
+  const char *filename = argv[optind];
+  simdjson::padded_string p;
   try {
-    p = get_corpus(filename);
-  } catch (const std::exception& e) { // caught by reference to base
+    simdjson::get_corpus(filename).swap(p);
+  } catch (const std::exception &e) { // caught by reference to base
     std::cout << "Could not load the file " << filename << std::endl;
     return EXIT_FAILURE;
   }
@@ -83,77 +81,101 @@ int main(int argc, char *argv[]) {
       std::cout << p.size() << " B ";
     std::cout << std::endl;
   }
-  char *buffer = allocate_padded_buffer(p.size() + 1);
+  char *buffer = simdjson::allocate_padded_buffer(p.size() + 1);
   memcpy(buffer, p.data(), p.size());
   buffer[p.size()] = '\0';
 
   int repeat = 50;
   int volume = p.size();
-  if(justdata) {
-    printf("name cycles_per_byte cycles_per_byte_err  gb_per_s gb_per_s_err \n");
+  if (just_data) {
+    printf(
+        "name cycles_per_byte cycles_per_byte_err  gb_per_s gb_per_s_err \n");
   }
-  size_t strlength = rapidstringme((char *)p.data()).size();
+  size_t strlength = rapid_stringme((char *)p.data()).size();
   if (verbose)
     std::cout << "input length is " << p.size() << " stringified length is "
               << strlength << std::endl;
-  BEST_TIME_NOCHECK("despacing with RapidJSON", rapidstringme((char *)p.data()), , repeat, volume, !justdata);
-  BEST_TIME_NOCHECK("despacing with RapidJSON Insitu", rapidstringmeInsitu((char *)buffer),
-                    memcpy(buffer, p.data(), p.size()), repeat, volume, !justdata);
+  BEST_TIME_NOCHECK("despacing with RapidJSON",
+                    rapid_stringme((char *)p.data()), , repeat, volume,
+                    !just_data);
+  BEST_TIME_NOCHECK(
+      "despacing with RapidJSON Insitu", rapid_stringme_insitu((char *)buffer),
+      memcpy(buffer, p.data(), p.size()), repeat, volume, !just_data);
   memcpy(buffer, p.data(), p.size());
 
-  size_t outlength =
-      jsonminify((const uint8_t *)buffer, p.size(), (uint8_t *)buffer);
+  size_t outlength = simdjson::json_minify((const uint8_t *)buffer, p.size(),
+                                           (uint8_t *)buffer);
   if (verbose)
-    std::cout << "jsonminify length is " << outlength << std::endl;
+    std::cout << "json_minify length is " << outlength << std::endl;
 
   uint8_t *cbuffer = (uint8_t *)buffer;
-  BEST_TIME("jsonminify", jsonminify(cbuffer, p.size(), cbuffer), outlength,
-            memcpy(buffer, p.data(), p.size()), repeat, volume, !justdata);
-  printf("minisize = %zu, original size = %zu  (minified down to %.2f percent of original) \n", outlength, p.size(), outlength * 100.0 / p.size());
+  BEST_TIME("json_minify", simdjson::json_minify(cbuffer, p.size(), cbuffer),
+            outlength, memcpy(buffer, p.data(), p.size()), repeat, volume,
+            !just_data);
+  printf("minisize = %zu, original size = %zu  (minified down to %.2f percent "
+         "of original) \n",
+         outlength, p.size(), outlength * 100.0 / p.size());
 
   /***
    * Is it worth it to minify before parsing?
    ***/
   rapidjson::Document d;
-  BEST_TIME("RapidJSON Insitu orig", d.ParseInsitu(buffer).HasParseError(), false,
-            memcpy(buffer, p.data(), p.size()), repeat, volume, !justdata);
+  BEST_TIME("RapidJSON Insitu orig", d.ParseInsitu(buffer).HasParseError(),
+            false, memcpy(buffer, p.data(), p.size()), repeat, volume,
+            !just_data);
 
-  char *minibuffer = allocate_padded_buffer(p.size() + 1);
-  size_t minisize = jsonminify((const uint8_t *)p.data(), p.size(), (uint8_t*) minibuffer);
-  minibuffer[minisize] = '\0';
+  char *mini_buffer = simdjson::allocate_padded_buffer(p.size() + 1);
+  size_t minisize = simdjson::json_minify((const uint8_t *)p.data(), p.size(),
+                                          (uint8_t *)mini_buffer);
+  mini_buffer[minisize] = '\0';
 
-  BEST_TIME("RapidJSON Insitu despaced", d.ParseInsitu(buffer).HasParseError(), false,
-            memcpy(buffer, minibuffer, p.size()),
-            repeat, volume, !justdata);
+  BEST_TIME("RapidJSON Insitu despaced", d.ParseInsitu(buffer).HasParseError(),
+            false, memcpy(buffer, mini_buffer, p.size()), repeat, volume,
+            !just_data);
 
-  size_t astbuffersize = p.size() * 2;
-  size_t * ast_buffer = (size_t *) malloc(astbuffersize * sizeof(size_t));
+  size_t ast_buffer_size = p.size() * 2;
+  size_t *ast_buffer = (size_t *)malloc(ast_buffer_size * sizeof(size_t));
 
-  BEST_TIME("sajson orig", sajson::parse(sajson::bounded_allocation(ast_buffer, astbuffersize), sajson::mutable_string_view(p.size(), buffer)).is_valid(), true, memcpy(buffer, p.data(), p.size()), repeat, volume, !justdata);
+  BEST_TIME(
+      "sajson orig",
+      sajson::parse(sajson::bounded_allocation(ast_buffer, ast_buffer_size),
+                    sajson::mutable_string_view(p.size(), buffer))
+          .is_valid(),
+      true, memcpy(buffer, p.data(), p.size()), repeat, volume, !just_data);
 
+  BEST_TIME(
+      "sajson despaced",
+      sajson::parse(sajson::bounded_allocation(ast_buffer, ast_buffer_size),
+                    sajson::mutable_string_view(minisize, buffer))
+          .is_valid(),
+      true, memcpy(buffer, mini_buffer, p.size()), repeat, volume, !just_data);
 
-  BEST_TIME("sajson despaced", sajson::parse(sajson::bounded_allocation(ast_buffer, astbuffersize), sajson::mutable_string_view(minisize, buffer)).is_valid(), true, memcpy(buffer, minibuffer, p.size()), repeat, volume, !justdata);
-
-  ParsedJson pj;
-  bool isallocok = pj.allocateCapacity(p.size(), 1024);
-  if(!isallocok) {
+  simdjson::ParsedJson pj;
+  bool is_alloc_ok = pj.allocate_capacity(p.size(), 1024);
+  if (!is_alloc_ok) {
     fprintf(stderr, "failed to allocate memory\n");
     return EXIT_FAILURE;
-  } 
-  BEST_TIME("simdjson orig", json_parse((const uint8_t*)buffer, p.size(), pj), true, memcpy(buffer, p.data(), p.size()), repeat, volume, !justdata);
-  
-  ParsedJson pj2;
-  bool isallocok2 = pj2.allocateCapacity(p.size(), 1024);
-  if(!isallocok2) {
+  }
+  bool automated_reallocation = false;
+  BEST_TIME("simdjson orig",
+            simdjson::json_parse((const uint8_t *)buffer, p.size(), pj,
+                                 automated_reallocation),
+            true, memcpy(buffer, p.data(), p.size()), repeat, volume,
+            !just_data);
+
+  simdjson::ParsedJson pj2;
+  bool is_alloc_ok2 = pj2.allocate_capacity(p.size(), 1024);
+  if (!is_alloc_ok2) {
     fprintf(stderr, "failed to allocate memory\n");
     return EXIT_FAILURE;
-  } 
-
-  BEST_TIME("simdjson despaced", json_parse((const uint8_t*)buffer, minisize, pj2), true, memcpy(buffer, minibuffer, p.size()), repeat, volume, !justdata);
-  aligned_free((void*)p.data());
+  }
+  automated_reallocation = false;
+  BEST_TIME("simdjson despaced",
+            simdjson::json_parse((const uint8_t *)buffer, minisize, pj2,
+                                 automated_reallocation),
+            true, memcpy(buffer, mini_buffer, p.size()), repeat, volume,
+            !just_data);
   free(buffer);
   free(ast_buffer);
-  free(minibuffer);
-
-
+  free(mini_buffer);
 }

benchmark/parse.cpp

@@ -3,9 +3,6 @@
 #ifndef _MSC_VER
 #include <dirent.h>
 #include <unistd.h>
-#include <x86intrin.h>
-#else
-#include <intrin.h>
 #endif
 #include <cinttypes>
 
@@ -31,26 +28,109 @@
 #endif
 //#define DEBUG
 #include "simdjson/common_defs.h"
+#include "simdjson/isadetection.h"
 #include "simdjson/jsonioutil.h"
 #include "simdjson/jsonparser.h"
 #include "simdjson/parsedjson.h"
 #include "simdjson/stage1_find_marks.h"
 #include "simdjson/stage2_build_tape.h"
-using namespace std;
+namespace simdjson {
+Architecture _find_best_supported_implementation() {
+  constexpr uint32_t haswell_flags =
+      instruction_set::AVX2 | instruction_set::PCLMULQDQ |
+      instruction_set::BMI1 | instruction_set::BMI2;
+  constexpr uint32_t westmere_flags =
+      instruction_set::SSE42 | instruction_set::PCLMULQDQ;
+  uint32_t supports = detect_supported_architectures();
+  // Order from best to worst (within architecture)
+  if ((haswell_flags & supports) == haswell_flags) {
+    return Architecture::HASWELL;
+  }
+  if ((westmere_flags & supports) == westmere_flags) {
+    return Architecture::WESTMERE;
+  }
+  if (instruction_set::NEON)
+    return Architecture::ARM64;
+
+  return Architecture::NONE;
+}
+
+using unified_functype = int(const uint8_t *buf, size_t len, ParsedJson &pj);
+using stage1_functype = int(const uint8_t *buf, size_t len, ParsedJson &pj);
+
+extern unified_functype *unified_ptr;
+
+extern stage1_functype *stage1_ptr;
+
+int unified_machine_dispatch(const uint8_t *buf, size_t len, ParsedJson &pj) {
+  Architecture best_implementation = _find_best_supported_implementation();
+  // Selecting the best implementation
+  switch (best_implementation) {
+#ifdef IS_X86_64
+  case Architecture::HASWELL:
+    unified_ptr = &unified_machine<Architecture::HASWELL>;
+    break;
+  case Architecture::WESTMERE:
+    unified_ptr = &unified_machine<Architecture::WESTMERE>;
+    break;
+#endif
+#ifdef IS_ARM64
+  case Architecture::ARM64:
+    unified_ptr = &unified_machine<Architecture::ARM64>;
+    break;
+#endif
+  default:
+    std::cerr << "The processor is not supported by simdjson." << std::endl;
+    return simdjson::UNEXPECTED_ERROR;
+  }
+
+  return unified_ptr(buf, len, pj);
+}
+
+// Responsible to select the best json_parse implementation
+int find_structural_bits_dispatch(const uint8_t *buf, size_t len,
+                                  ParsedJson &pj) {
+  Architecture best_implementation = _find_best_supported_implementation();
+  // Selecting the best implementation
+  switch (best_implementation) {
+#ifdef IS_X86_64
+  case Architecture::HASWELL:
+    stage1_ptr = &find_structural_bits<Architecture::HASWELL>;
+    break;
+  case Architecture::WESTMERE:
+    stage1_ptr = &find_structural_bits<Architecture::WESTMERE>;
+    break;
+#endif
+#ifdef IS_ARM64
+  case Architecture::ARM64:
+    stage1_ptr = &find_structural_bits<Architecture::ARM64>;
+    break;
+#endif
+  default:
+    std::cerr << "The processor is not supported by simdjson." << std::endl;
+    return simdjson::UNEXPECTED_ERROR;
+  }
+
+  return stage1_ptr(buf, len, pj);
+}
+
+stage1_functype *stage1_ptr = &find_structural_bits_dispatch;
+unified_functype *unified_ptr = &unified_machine_dispatch;
+} // namespace simdjson
 
 int main(int argc, char *argv[]) {
   bool verbose = false;
   bool dump = false;
-  bool jsonoutput = false;
-  bool forceoneiteration = false;
-  bool justdata = false;
+  bool json_output = false;
+  bool force_one_iteration = false;
+  bool just_data = false;
 #ifndef _MSC_VER
   int c;
 
   while ((c = getopt(argc, argv, "1vdt")) != -1) {
     switch (c) {
     case 't':
-      justdata = true;
+      just_data = true;
       break;
     case 'v':
       verbose = true;
@@ -59,65 +139,84 @@ int main(int argc, char *argv[]) {
       dump = true;
       break;
     case 'j':
-      jsonoutput = true;
+      json_output = true;
       break;
     case '1':
-      forceoneiteration = true;
+      force_one_iteration = true;
       break;
     default:
       abort();
     }
-}
+  }
 #else
   int optind = 1;
 #endif
   if (optind >= argc) {
-    cerr << "Usage: " << argv[0] << " <jsonfile>" << endl;
+    std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
     exit(1);
   }
   const char *filename = argv[optind];
   if (optind + 1 < argc) {
-    cerr << "warning: ignoring everything after " << argv[optind + 1] << endl;
+    std::cerr << "warning: ignoring everything after " << argv[optind + 1]
+              << std::endl;
   }
   if (verbose) {
-    cout << "[verbose] loading " << filename << endl;
-}
-  std::string_view p;
+    std::cout << "[verbose] loading " << filename << std::endl;
+  }
+  simdjson::padded_string p;
   try {
-    p = get_corpus(filename);
-  } catch (const std::exception &e) { // caught by reference to base
+    simdjson::get_corpus(filename).swap(p);
+  } catch (const std::exception &) { // caught by reference to base
     std::cout << "Could not load the file " << filename << std::endl;
     return EXIT_FAILURE;
   }
   if (verbose) {
-    cout << "[verbose] loaded " << filename << " (" << p.size() << " bytes)"
-         << endl;
-}
+    std::cout << "[verbose] loaded " << filename << " (" << p.size()
+              << " bytes)" << std::endl;
+  }
 #if defined(DEBUG)
   const uint32_t iterations = 1;
 #else
   const uint32_t iterations =
-      forceoneiteration ? 1 : (p.size() < 1 * 1000 * 1000 ? 1000 : 10);
+      force_one_iteration ? 1 : (p.size() < 1 * 1000 * 1000 ? 1000 : 10);
 #endif
-  vector<double> res;
+  std::vector<double> res;
   res.resize(iterations);
-
+  if (!just_data)
+    printf("number of iterations %u \n", iterations);
 #if !defined(__linux__)
 #define SQUASH_COUNTERS
-  if (justdata) {
-    printf("justdata (-t) flag only works under linux.\n");
+  if (just_data) {
+    printf("just_data (-t) flag only works under linux.\n");
   }
 #endif
-
+  { // practice run
+    simdjson::ParsedJson pj;
+    bool allocok = pj.allocate_capacity(p.size());
+    if (allocok) {
+      simdjson::stage1_ptr((const uint8_t *)p.data(), p.size(), pj);
+      simdjson::unified_ptr(
+          (const uint8_t
+               *)(const uint8_t
+                      *)(const uint8_t
+                             *)(const uint8_t
+                                    *)(const uint8_t
+                                           *)(const uint8_t
+                                                  *)(const uint8_t
+                                                         *)(const uint8_t *)
+              p.data(),
+          p.size(), pj);
+    }
+  }
 #ifndef SQUASH_COUNTERS
-  vector<int> evts;
+  std::vector<int> evts;
   evts.push_back(PERF_COUNT_HW_CPU_CYCLES);
   evts.push_back(PERF_COUNT_HW_INSTRUCTIONS);
   evts.push_back(PERF_COUNT_HW_BRANCH_MISSES);
   evts.push_back(PERF_COUNT_HW_CACHE_REFERENCES);
   evts.push_back(PERF_COUNT_HW_CACHE_MISSES);
   LinuxEvents<PERF_TYPE_HARDWARE> unified(evts);
-  vector<unsigned long long> results;
+  std::vector<unsigned long long> results;
   results.resize(evts.size());
   unsigned long cy0 = 0, cy1 = 0, cy2 = 0;
   unsigned long cl0 = 0, cl1 = 0, cl2 = 0;
@@ -126,38 +225,30 @@ int main(int argc, char *argv[]) {
   unsigned long cmis0 = 0, cmis1 = 0, cmis2 = 0;
 #endif
   bool isok = true;
-
+#ifndef SQUASH_COUNTERS
   for (uint32_t i = 0; i < iterations; i++) {
     if (verbose) {
-      cout << "[verbose] iteration # " << i << endl;
-}
-#ifndef SQUASH_COUNTERS
+      std::cout << "[verbose] iteration # " << i << std::endl;
+    }
     unified.start();
-#endif
-    ParsedJson pj;
-    bool allocok = pj.allocateCapacity(p.size());
+    simdjson::ParsedJson pj;
+    bool allocok = pj.allocate_capacity(p.size());
     if (!allocok) {
       std::cerr << "failed to allocate memory" << std::endl;
       return EXIT_FAILURE;
     }
-#ifndef SQUASH_COUNTERS
     unified.end(results);
     cy0 += results[0];
     cl0 += results[1];
     mis0 += results[2];
     cref0 += results[3];
     cmis0 += results[4];
-#endif
     if (verbose) {
-      cout << "[verbose] allocated memory for parsed JSON " << endl;
-}
-
-    auto start = std::chrono::steady_clock::now();
-#ifndef SQUASH_COUNTERS
+      std::cout << "[verbose] allocated memory for parsed JSON " << std::endl;
+    }
     unified.start();
-#endif
-    isok = find_structural_bits(p.data(), p.size(), pj);
-#ifndef SQUASH_COUNTERS
+    isok = (simdjson::stage1_ptr((const uint8_t *)p.data(), p.size(), pj) ==
+            simdjson::SUCCESS);
     unified.end(results);
     cy1 += results[0];
     cl1 += results[1];
@@ -165,14 +256,13 @@ int main(int argc, char *argv[]) {
     cref1 += results[3];
     cmis1 += results[4];
     if (!isok) {
-      cout << "Failed out during stage 1\n";
+      std::cout << "Failed during stage 1" << std::endl;
       break;
     }
     unified.start();
-#endif
-
-    isok = isok && !unified_machine(p.data(), p.size(), pj);
-#ifndef SQUASH_COUNTERS
+    isok = isok &&
+           (simdjson::SUCCESS ==
+            simdjson::unified_ptr((const uint8_t *)p.data(), p.size(), pj));
     unified.end(results);
     cy2 += results[0];
     cl2 += results[1];
@@ -180,30 +270,61 @@ int main(int argc, char *argv[]) {
     cref2 += results[3];
     cmis2 += results[4];
     if (!isok) {
-      cout << "Failed out during stage 2\n";
+      std::cout << "Failed during stage 2" << std::endl;
       break;
     }
+  }
 #endif
+  // we do it again, this time just measuring the elapsed time
+  for (uint32_t i = 0; i < iterations; i++) {
+    if (verbose) {
+      std::cout << "[verbose] iteration # " << i << std::endl;
+    }
+    simdjson::ParsedJson pj;
+    bool allocok = pj.allocate_capacity(p.size());
+    if (!allocok) {
+      std::cerr << "failed to allocate memory" << std::endl;
+      return EXIT_FAILURE;
+    }
+    if (verbose) {
+      std::cout << "[verbose] allocated memory for parsed JSON " << std::endl;
+    }
 
+    auto start = std::chrono::steady_clock::now();
+    isok = (simdjson::stage1_ptr((const uint8_t *)p.data(), p.size(), pj) ==
+            simdjson::SUCCESS);
+    isok = isok &&
+           (simdjson::SUCCESS ==
+            simdjson::unified_ptr((const uint8_t *)p.data(), p.size(), pj));
     auto end = std::chrono::steady_clock::now();
     std::chrono::duration<double> secs = end - start;
     res[i] = secs.count();
+    if (!isok) {
+      std::cerr << pj.get_error_message() << std::endl;
+      std::cerr << "Could not parse. " << std::endl;
+      return EXIT_FAILURE;
+    }
   }
-  ParsedJson pj = build_parsed_json(p); // do the parsing again to get the stats
-  if (!pj.isValid()) {
+  simdjson::ParsedJson pj =
+      build_parsed_json(p); // do the parsing again to get the stats
+  if (!pj.is_valid()) {
+    std::cerr << pj.get_error_message() << std::endl;
     std::cerr << "Could not parse. " << std::endl;
     return EXIT_FAILURE;
   }
+  double min_result = *min_element(res.begin(), res.end());
+  double speedinGBs = (p.size()) / (min_result * 1000000000.0);
 #ifndef SQUASH_COUNTERS
   unsigned long total = cy0 + cy1 + cy2;
-  if (justdata) {
+  if (just_data) {
     float cpb0 = (double)cy0 / (iterations * p.size());
     float cpb1 = (double)cy1 / (iterations * p.size());
     float cpb2 = (double)cy2 / (iterations * p.size());
     float cpbtotal = (double)total / (iterations * p.size());
     char *newfile = (char *)malloc(strlen(filename) + 1);
-    if (newfile == NULL)
+    if (newfile == NULL) {
       return EXIT_FAILURE;
+    }
     ::strcpy(newfile, filename);
     char *snewfile = ::basename(newfile);
     size_t nl = strlen(snewfile);
@@ -213,8 +334,8 @@ int main(int argc, char *argv[]) {
         break;
       }
     }
-    printf("\"%s\"\t%f\t%f\t%f\t%f\n", snewfile, cpb0, cpb1, cpb2,
-           cpbtotal);
+    printf("\"%s\"\t%f\t%f\t%f\t%f\t%f\n", snewfile, cpb0, cpb1, cpb2, cpbtotal,
+           speedinGBs);
     free(newfile);
   } else {
     printf("number of bytes %ld number of structural chars %u ratio %.3f\n",
@@ -250,21 +371,20 @@ int main(int argc, char *argv[]) {
 
     printf(" all stages: %.2f cycles per input byte.\n",
            (double)total / (iterations * p.size()));
+    printf("Estimated average frequency: %.3f GHz.\n",
+           (double)total / (iterations * min_result * 1000000000.0));
   }
 #endif
-  double min_result = *min_element(res.begin(), res.end());
-  if (!justdata) {
-    cout << "Min:  " << min_result << " bytes read: " << p.size()
-         << " Gigabytes/second: " << (p.size()) / (min_result * 1000000000.0)
-         << "\n";
-}
-  if (jsonoutput) {
-    isok = isok && pj.printjson(std::cout);
+  if (!just_data) {
+    std::cout << "Min:  " << min_result << " bytes read: " << p.size()
+              << " Gigabytes/second: " << speedinGBs << std::endl;
+  }
+  if (json_output) {
+    isok = isok && pj.print_json(std::cout);
   }
   if (dump) {
     isok = isok && pj.dump_raw_tape(std::cout);
   }
-  aligned_free((void *)p.data());
   if (!isok) {
     fprintf(stderr, " Parsing failed. \n ");
     return EXIT_FAILURE;

benchmark/parseandstatcompetition.cpp

@@ -12,8 +12,7 @@
 #include "sajson.h"
 
 using namespace rapidjson;
-using namespace std;
-
+using namespace simdjson;
 struct stat_s {
   size_t number_count;
   size_t object_count;
@@ -44,11 +43,11 @@ void print_stat(const stat_t &s) {
          s.true_count, s.false_count);
 }
 
-__attribute__ ((noinline))
-stat_t simdjson_computestats(const std::string_view &p) {
+__attribute__((noinline)) stat_t
+simdjson_compute_stats(const simdjson::padded_string &p) {
   stat_t answer;
-  ParsedJson pj = build_parsed_json(p);
-  answer.valid = pj.isValid();
+  simdjson::ParsedJson pj = build_parsed_json(p);
+  answer.valid = pj.is_valid();
   if (!answer.valid) {
     return answer;
   }
@@ -58,24 +57,24 @@ stat_t simdjson_computestats(const std::string_view &p) {
   answer.null_count = 0;
   answer.true_count = 0;
   answer.false_count = 0;
-  size_t tapeidx = 0;
-  uint64_t tape_val = pj.tape[tapeidx++];
+  size_t tape_idx = 0;
+  uint64_t tape_val = pj.tape[tape_idx++];
   uint8_t type = (tape_val >> 56);
-  size_t howmany = 0;
+  size_t how_many = 0;
   assert(type == 'r');
-  howmany = tape_val & JSONVALUEMASK;
-  for (; tapeidx < howmany; tapeidx++) {
-    tape_val = pj.tape[tapeidx];
-    // uint64_t payload = tape_val & JSONVALUEMASK;
+  how_many = tape_val & JSON_VALUE_MASK;
+  for (; tape_idx < how_many; tape_idx++) {
+    tape_val = pj.tape[tape_idx];
+    // uint64_t payload = tape_val & JSON_VALUE_MASK;
     type = (tape_val >> 56);
     switch (type) {
     case 'l': // we have a long int
       answer.number_count++;
-      tapeidx++; // skipping the integer
+      tape_idx++; // skipping the integer
       break;
     case 'd': // we have a double
       answer.number_count++;
-      tapeidx++; // skipping the double
+      tape_idx++; // skipping the double
       break;
     case 'n': // we have a null
       answer.null_count++;
@@ -146,8 +145,8 @@ void sajson_traverse(stat_t &stats, const sajson::value &node) {
   }
 }
 
-__attribute__ ((noinline))
-stat_t sasjon_computestats(const std::string_view &p) {
+__attribute__((noinline)) stat_t
+sasjon_compute_stats(const simdjson::padded_string &p) {
   stat_t answer;
   char *buffer = (char *)malloc(p.size());
   memcpy(buffer, p.data(), p.size());
@@ -204,8 +203,8 @@ void rapid_traverse(stat_t &stats, const rapidjson::Value &v) {
   }
 }
 
-__attribute__ ((noinline))
-stat_t rapid_computestats(const std::string_view &p) {
+__attribute__((noinline)) stat_t
+rapid_compute_stats(const simdjson::padded_string &p) {
   stat_t answer;
   char *buffer = (char *)malloc(p.size() + 1);
   memcpy(buffer, p.data(), p.size());
@@ -229,13 +228,13 @@ stat_t rapid_computestats(const std::string_view &p) {
 
 int main(int argc, char *argv[]) {
   bool verbose = false;
-  bool justdata = false;
+  bool just_data = false;
 
   int c;
   while ((c = getopt(argc, argv, "vt")) != -1)
     switch (c) {
     case 't':
-      justdata = true;
+      just_data = true;
       break;
     case 'v':
       verbose = true;
@@ -244,19 +243,22 @@ int main(int argc, char *argv[]) {
       abort();
     }
   if (optind >= argc) {
-    cerr << "Using different parsers, we compute the content statistics of "
-            "JSON documents.\n";
-    cerr << "Usage: " << argv[0] << " <jsonfile>\n";
-    cerr << "Or " << argv[0] << " -v <jsonfile>\n";
+    std::cerr
+        << "Using different parsers, we compute the content statistics of "
+           "JSON documents."
+        << std::endl;
+    std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
+    std::cerr << "Or " << argv[0] << " -v <jsonfile>" << std::endl;
     exit(1);
   }
   const char *filename = argv[optind];
   if (optind + 1 < argc) {
-    cerr << "warning: ignoring everything after " << argv[optind + 1] << endl;
+    std::cerr << "warning: ignoring everything after " << argv[optind + 1]
+              << std::endl;
   }
-  std::string_view p;
+  simdjson::padded_string p;
   try {
-    p = get_corpus(filename);
+    simdjson::get_corpus(filename).swap(p);
   } catch (const std::exception &e) { // caught by reference to base
     std::cout << "Could not load the file " << filename << std::endl;
     return EXIT_FAILURE;
@@ -272,17 +274,17 @@ int main(int argc, char *argv[]) {
       std::cout << p.size() << " B ";
     std::cout << std::endl;
   }
-  stat_t s1 = simdjson_computestats(p);
+  stat_t s1 = simdjson_compute_stats(p);
   if (verbose) {
     printf("simdjson: ");
     print_stat(s1);
   }
-  stat_t s2 = rapid_computestats(p);
+  stat_t s2 = rapid_compute_stats(p);
   if (verbose) {
     printf("rapid:    ");
     print_stat(s2);
   }
-  stat_t s3 = sasjon_computestats(p);
+  stat_t s3 = sasjon_compute_stats(p);
   if (verbose) {
     printf("sasjon:   ");
     print_stat(s3);
@@ -291,14 +293,13 @@ int main(int argc, char *argv[]) {
   assert(stat_equal(s1, s3));
   int repeat = 50;
   int volume = p.size();
-  if(justdata) {
-    printf("name cycles_per_byte cycles_per_byte_err  gb_per_s gb_per_s_err \n");
+  if (just_data) {
+    printf("name cycles_per_byte cycles_per_byte_err gb_per_s gb_per_s_err \n");
   }
-  BEST_TIME("simdjson  ", simdjson_computestats(p).valid, true, , repeat,
-            volume, !justdata);
-  BEST_TIME("RapidJSON  ", rapid_computestats(p).valid, true, , repeat, volume,
-            !justdata);
-  BEST_TIME("sasjon  ", sasjon_computestats(p).valid, true, , repeat, volume,
-            !justdata);
-  aligned_free((void*)p.data());
+  BEST_TIME("simdjson  ", simdjson_compute_stats(p).valid, true, , repeat,
+            volume, !just_data);
+  BEST_TIME("RapidJSON  ", rapid_compute_stats(p).valid, true, , repeat, volume,
+            !just_data);
+  BEST_TIME("sasjon  ", sasjon_compute_stats(p).valid, true, , repeat, volume,
+            !just_data);
 }

benchmark/parsingcompetition.cpp

@@ -1,7 +1,7 @@
 #include "simdjson/jsonparser.h"
 #ifndef _MSC_VER
-#include <unistd.h>
 #include "linux-perf-events.h"
+#include <unistd.h>
 #ifdef __linux__
 #include <libgen.h>
 #endif //__linux__
@@ -11,7 +11,6 @@
 
 #include "benchmark.h"
 
-
 // #define RAPIDJSON_SSE2 // bad for performance
 // #define RAPIDJSON_SSE42 // bad for performance
 #include "rapidjson/document.h"
@@ -21,8 +20,10 @@
 
 #include "sajson.h"
 
-#ifdef ALLPARSER
+#include <nlohmann/json.hpp>
+using json = nlohmann::json;
 
+#ifdef ALLPARSER
 
 #include "fastjson.cpp"
 #include "fastjson_dom.cpp"
@@ -30,22 +31,20 @@
 
 #include "json11.cpp"
 extern "C" {
+#include "cJSON.c"
+#include "cJSON.h"
+#include "jsmn.c"
+#include "jsmn.h"
 #include "ujdecode.h"
 #include "ultrajsondec.c"
-#include "cJSON.h"
-#include "cJSON.c"
-#include "jsmn.h"
-#include "jsmn.c"
 }
 
-#include "json/json.h"
 #include "jsoncpp.cpp"
+#include "json/json.h"
 
-#endif 
+#endif
 
 using namespace rapidjson;
-using namespace std;
-
 
 #ifdef ALLPARSER
 // fastjson has a tricky interface
@@ -63,12 +62,12 @@ bool fastjson_parse(const char *input) {
 
 int main(int argc, char *argv[]) {
   bool verbose = false;
-  bool justdata = false;
+  bool just_data = false;
   int c;
   while ((c = getopt(argc, argv, "vt")) != -1)
     switch (c) {
     case 't':
-      justdata = true;
+      just_data = true;
       break;
     case 'v':
       verbose = true;
@@ -77,19 +76,19 @@ int main(int argc, char *argv[]) {
       abort();
     }
   if (optind >= argc) {
-    cerr << "Usage: " << argv[0] << " <jsonfile>\n";
-    cerr << "Or " << argv[0] << " -v <jsonfile>\n";
-    cerr << "To enable parsers that are not standard compliant, use the -a "
-            "flag\n";
+    std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
+    std::cerr << "Or " << argv[0] << " -v <jsonfile>" << std::endl;
+    std::cerr << "The '-t' flag outputs a table. " << std::endl;
     exit(1);
   }
   const char *filename = argv[optind];
   if (optind + 1 < argc) {
-    cerr << "warning: ignoring everything after " << argv[optind + 1] << endl;
+    std::cerr << "warning: ignoring everything after " << argv[optind + 1]
+              << std::endl;
   }
-  std::string_view p;
+  simdjson::padded_string p;
   try {
-    p = get_corpus(filename);
+    simdjson::get_corpus(filename).swap(p);
   } catch (const std::exception &e) { // caught by reference to base
     std::cout << "Could not load the file " << filename << std::endl;
     return EXIT_FAILURE;
@@ -105,162 +104,195 @@ int main(int argc, char *argv[]) {
       std::cout << p.size() << " B ";
     std::cout << std::endl;
   }
-  ParsedJson pj;
-  bool allocok = pj.allocateCapacity(p.size(), 1024);
+  simdjson::ParsedJson pj;
+  bool allocok = pj.allocate_capacity(p.size(), 1024);
 
   if (!allocok) {
     std::cerr << "can't allocate memory" << std::endl;
     return EXIT_FAILURE;
   }
-  int repeat = 50;
+  int repeat = (p.size() < 1 * 1000 * 1000 ? 1000 : 10);
   int volume = p.size();
-  if(justdata) {
-    printf("name cycles_per_byte cycles_per_byte_err  gb_per_s gb_per_s_err \n");
+  if (just_data) {
+    printf("%-42s %20s %20s %20s %20s \n", "name", "cycles_per_byte",
+           "cycles_per_byte_err", "gb_per_s", "gb_per_s_err");
   }
-  if(!justdata) BEST_TIME("simdjson (dynamic mem) ", build_parsed_json(p).isValid(), true, ,
-            repeat, volume, !justdata);
-  // (static alloc) 
-  BEST_TIME("simdjson ", json_parse(p, pj), simdjson::SUCCESS, , repeat,
-            volume, !justdata);
+  if (!just_data)
+    BEST_TIME("simdjson (dynamic mem) ", build_parsed_json(p).is_valid(), true,
+              , repeat, volume, !just_data);
+  // (static alloc)
+  BEST_TIME("simdjson ", json_parse(p, pj), simdjson::SUCCESS, , repeat, volume,
+            !just_data);
 
- 
   rapidjson::Document d;
 
   char *buffer = (char *)malloc(p.size() + 1);
   memcpy(buffer, p.data(), p.size());
   buffer[p.size()] = '\0';
 #ifndef ALLPARSER
-  if(!justdata) 
-#endif 
-  BEST_TIME(
-      "RapidJSON  ",
-      d.Parse<kParseValidateEncodingFlag>((const char *)buffer).HasParseError(),
-      false, memcpy(buffer, p.data(), p.size()), repeat, volume, !justdata);
+  if (!just_data)
+#endif
+    BEST_TIME("RapidJSON  ",
+              d.Parse<kParseValidateEncodingFlag>((const char *)buffer)
+                  .HasParseError(),
+              false, memcpy(buffer, p.data(), p.size()), repeat, volume,
+              !just_data);
   BEST_TIME("RapidJSON (insitu)",
             d.ParseInsitu<kParseValidateEncodingFlag>(buffer).HasParseError(),
-            false, memcpy(buffer, p.data(), p.size()) && (buffer[p.size()] = '\0'), repeat, volume, !justdata);
+            false,
+            memcpy(buffer, p.data(), p.size()) && (buffer[p.size()] = '\0'),
+            repeat, volume, !just_data);
 #ifndef ALLPARSER
-  if(!justdata) 
-#endif 
-  BEST_TIME("sajson (dynamic mem)",
-            sajson::parse(sajson::dynamic_allocation(),
-                          sajson::mutable_string_view(p.size(), buffer))
-                .is_valid(),
-            true, memcpy(buffer, p.data(), p.size()), repeat, volume, !justdata);
+  if (!just_data)
+#endif
+    BEST_TIME("sajson (dynamic mem)",
+              sajson::parse(sajson::dynamic_allocation(),
+                            sajson::mutable_string_view(p.size(), buffer))
+                  .is_valid(),
+              true, memcpy(buffer, p.data(), p.size()), repeat, volume,
+              !just_data);
 
-  size_t astbuffersize = p.size();
-  size_t *ast_buffer = (size_t *)malloc(astbuffersize * sizeof(size_t));
+  size_t ast_buffer_size = p.size();
+  size_t *ast_buffer = (size_t *)malloc(ast_buffer_size * sizeof(size_t));
   //  (static alloc, insitu)
-  BEST_TIME("sajson",
-            sajson::parse(sajson::bounded_allocation(ast_buffer, astbuffersize),
-                          sajson::mutable_string_view(p.size(), buffer))
-                .is_valid(),
-            true, memcpy(buffer, p.data(), p.size()), repeat, volume, !justdata);
-#ifdef __linux__
-  if(!justdata) {
-      vector<int> evts;
-      evts.push_back(PERF_COUNT_HW_CPU_CYCLES);
-      evts.push_back(PERF_COUNT_HW_INSTRUCTIONS);
-      evts.push_back(PERF_COUNT_HW_BRANCH_MISSES);
-      evts.push_back(PERF_COUNT_HW_CACHE_REFERENCES);
-      evts.push_back(PERF_COUNT_HW_CACHE_MISSES);
-      LinuxEvents<PERF_TYPE_HARDWARE> unified(evts);
-      vector<unsigned long long> results;
-      vector<unsigned long long> stats;
-      results.resize(evts.size());
-      stats.resize(evts.size());
-      std::fill(stats.begin(), stats.end(), 0);// unnecessary
-      for(int i = 0; i < repeat; i++) {
-        unified.start();
-        if(json_parse(p, pj) != true) printf("bug\n");
-        unified.end(results);
-        std::transform (stats.begin(), stats.end(), results.begin(), stats.begin(), std::plus<unsigned long long>());
-      }
-      printf("simdjson : cycles %10.0f instructions %10.0f branchmisses %10.0f cacheref %10.0f cachemisses %10.0f  bytespercachemiss %10.0f inspercycle %10.1f insperbyte %10.1f\n",
-      stats[0] * 1.0 / repeat, stats[1] * 1.0 / repeat, stats[2] * 1.0 / repeat, stats[3] * 1.0 / repeat,  stats[4] * 1.0 / repeat, volume * repeat * 1.0 / stats[2], stats[1] *1.0 / stats[0], stats[1] * 1.0 / (volume * repeat));
+  BEST_TIME(
+      "sajson",
+      sajson::parse(sajson::bounded_allocation(ast_buffer, ast_buffer_size),
+                    sajson::mutable_string_view(p.size(), buffer))
+          .is_valid(),
+      true, memcpy(buffer, p.data(), p.size()), repeat, volume, !just_data);
+  size_t expected = json::parse(p.data(), p.data() + p.size()).size();
+  BEST_TIME("nlohmann-json", json::parse(buffer, buffer + p.size()).size(),
+            expected, memcpy(buffer, p.data(), p.size()), repeat, volume,
+            !just_data);
 
-      std::fill(stats.begin(), stats.end(), 0);
-      for(int i = 0; i < repeat; i++) {
-        memcpy(buffer, p.data(), p.size());
-        buffer[p.size()] = '\0';
-        unified.start();
-        if(d.ParseInsitu<kParseValidateEncodingFlag>(buffer).HasParseError() != false) printf("bug\n");
-        unified.end(results);
-        std::transform (stats.begin(), stats.end(), results.begin(), stats.begin(), std::plus<unsigned long long>());
-      }
-      printf("RapidJSON: cycles %10.0f instructions %10.0f branchmisses %10.0f cacheref %10.0f cachemisses %10.0f  bytespercachemiss %10.0f inspercycle %10.1f insperbyte %10.1f\n",
-      stats[0] * 1.0 / repeat, stats[1] * 1.0 / repeat, stats[2] * 1.0 / repeat, stats[3] * 1.0 / repeat,  stats[4] * 1.0 / repeat,  volume * repeat * 1.0 / stats[2], stats[1] *1.0 / stats[0], stats[1] * 1.0 / (volume * repeat));
-
-      std::fill(stats.begin(), stats.end(), 0);// unnecessary
-      for(int i = 0; i < repeat; i++) {
-        memcpy(buffer, p.data(), p.size());
-        unified.start();
-        if(sajson::parse(sajson::bounded_allocation(ast_buffer, astbuffersize),
-                          sajson::mutable_string_view(p.size(), buffer))
-                .is_valid() != true) printf("bug\n");
-        unified.end(results);
-        std::transform (stats.begin(), stats.end(), results.begin(), stats.begin(), std::plus<unsigned long long>());
-      }
-      printf("sajson   : cycles %10.0f instructions %10.0f branchmisses %10.0f cacheref %10.0f cachemisses %10.0f  bytespercachemiss %10.0f inspercycle %10.1f insperbyte %10.1f\n",
-      stats[0] * 1.0 / repeat, stats[1] * 1.0 / repeat, stats[2] * 1.0 / repeat, stats[3] * 1.0 / repeat,  stats[4] * 1.0 / repeat,  volume * repeat * 1.0 / stats[2], stats[1] *1.0 / stats[0], stats[1] * 1.0 / (volume * repeat));
-  }
-#endif//  __linux__
 #ifdef ALLPARSER
   std::string json11err;
-    BEST_TIME("dropbox (json11)     ",
-              ((json11::Json::parse(buffer, json11err).is_null()) ||
-               (!json11err.empty())),
-              false, memcpy(buffer, p.data(), p.size()), repeat, volume, !justdata);
+  BEST_TIME("dropbox (json11)     ",
+            ((json11::Json::parse(buffer, json11err).is_null()) ||
+             (!json11err.empty())),
+            false, memcpy(buffer, p.data(), p.size()), repeat, volume,
+            !just_data);
 
-    BEST_TIME("fastjson             ", fastjson_parse(buffer), true,
-              memcpy(buffer, p.data(), p.size()), repeat, volume, !justdata);
+  BEST_TIME("fastjson             ", fastjson_parse(buffer), true,
+            memcpy(buffer, p.data(), p.size()), repeat, volume, !just_data);
   JsonValue value;
   JsonAllocator allocator;
   char *endptr;
-    BEST_TIME("gason             ",
-              jsonParse(buffer, &endptr, &value, allocator), JSON_OK,
-              memcpy(buffer, p.data(), p.size()), repeat, volume, !justdata);
+  BEST_TIME("gason             ", jsonParse(buffer, &endptr, &value, allocator),
+            JSON_OK, memcpy(buffer, p.data(), p.size()), repeat, volume,
+            !just_data);
   void *state;
-    BEST_TIME("ultrajson         ",
-              (UJDecode(buffer, p.size(), NULL, &state) == NULL), false,
-              memcpy(buffer, p.data(), p.size()), repeat, volume, !justdata);
+  BEST_TIME("ultrajson         ",
+            (UJDecode(buffer, p.size(), NULL, &state) == NULL), false,
+            memcpy(buffer, p.data(), p.size()), repeat, volume, !just_data);
 
-
-
-  auto * tokens = make_unique<jsmntok_t[](p.size());
-  if(tokens == NULL) {
-    printf("Failed to alloc memory for jsmn\n");
-  } else {
+  {
+    std::unique_ptr<jsmntok_t[]> tokens =
+        std::make_unique<jsmntok_t[]>(p.size());
     jsmn_parser parser;
     jsmn_init(&parser);
     memcpy(buffer, p.data(), p.size());
     buffer[p.size()] = '\0';
-    BEST_TIME("jsmn           ",
-              (jsmn_parse(&parser, buffer, p.size(), tokens.get(), p.size()) > 0), true,
-              jsmn_init(&parser), repeat, volume, !justdata);
+    BEST_TIME(
+        "jsmn           ",
+        (jsmn_parse(&parser, buffer, p.size(), tokens.get(), p.size()) > 0),
+        true, jsmn_init(&parser), repeat, volume, !just_data);
   }
-
   memcpy(buffer, p.data(), p.size());
   buffer[p.size()] = '\0';
-  cJSON * tree = cJSON_Parse(buffer);
-  BEST_TIME("cJSON           ",
-              ((tree = cJSON_Parse(buffer)) != NULL ), true,
-               cJSON_Delete(tree), repeat, volume, !justdata);
+  cJSON *tree = cJSON_Parse(buffer);
+  BEST_TIME("cJSON           ", ((tree = cJSON_Parse(buffer)) != NULL), true,
+            cJSON_Delete(tree), repeat, volume, !just_data);
   cJSON_Delete(tree);
-  
+
   Json::CharReaderBuilder b;
-  Json::CharReader * jsoncppreader = b.newCharReader();
+  Json::CharReader *json_cpp_reader = b.newCharReader();
   Json::Value root;
   Json::String errs;
   BEST_TIME("jsoncpp           ",
-              jsoncppreader->parse(buffer,buffer+volume,&root,&errs), true,
-               , repeat, volume, !justdata);
-  delete jsoncppreader;
+            json_cpp_reader->parse(buffer, buffer + volume, &root, &errs), true,
+            , repeat, volume, !just_data);
+  delete json_cpp_reader;
 #endif
-  if(!justdata) BEST_TIME("memcpy            ",
-            (memcpy(buffer, p.data(), p.size()) == buffer), true, , repeat,
-            volume, !justdata);
-  aligned_free((void *)p.data());
+  if (!just_data)
+    BEST_TIME("memcpy            ",
+              (memcpy(buffer, p.data(), p.size()) == buffer), true, , repeat,
+              volume, !just_data);
+#ifdef __linux__
+  if (!just_data) {
+    printf("\n \n <doing additional analysis with performance counters (Linux "
+           "only)>\n");
+    std::vector<int> evts;
+    evts.push_back(PERF_COUNT_HW_CPU_CYCLES);
+    evts.push_back(PERF_COUNT_HW_INSTRUCTIONS);
+    evts.push_back(PERF_COUNT_HW_BRANCH_MISSES);
+    evts.push_back(PERF_COUNT_HW_CACHE_REFERENCES);
+    evts.push_back(PERF_COUNT_HW_CACHE_MISSES);
+    LinuxEvents<PERF_TYPE_HARDWARE> unified(evts);
+    std::vector<unsigned long long> results;
+    std::vector<unsigned long long> stats;
+    results.resize(evts.size());
+    stats.resize(evts.size());
+    std::fill(stats.begin(), stats.end(), 0); // unnecessary
+    for (int i = 0; i < repeat; i++) {
+      unified.start();
+      if (json_parse(p, pj) != simdjson::SUCCESS)
+        printf("bug\n");
+      unified.end(results);
+      std::transform(stats.begin(), stats.end(), results.begin(), stats.begin(),
+                     std::plus<unsigned long long>());
+    }
+    printf("simdjson : cycles %10.0f instructions %10.0f branchmisses %10.0f "
+           "cacheref %10.0f cachemisses %10.0f  bytespercachemiss %10.0f "
+           "inspercycle %10.1f insperbyte %10.1f\n",
+           stats[0] * 1.0 / repeat, stats[1] * 1.0 / repeat,
+           stats[2] * 1.0 / repeat, stats[3] * 1.0 / repeat,
+           stats[4] * 1.0 / repeat, volume * repeat * 1.0 / stats[2],
+           stats[1] * 1.0 / stats[0], stats[1] * 1.0 / (volume * repeat));
+
+    std::fill(stats.begin(), stats.end(), 0);
+    for (int i = 0; i < repeat; i++) {
+      memcpy(buffer, p.data(), p.size());
+      buffer[p.size()] = '\0';
+      unified.start();
+      if (d.ParseInsitu<kParseValidateEncodingFlag>(buffer).HasParseError() !=
+          false)
+        printf("bug\n");
+      unified.end(results);
+      std::transform(stats.begin(), stats.end(), results.begin(), stats.begin(),
+                     std::plus<unsigned long long>());
+    }
+    printf("RapidJSON: cycles %10.0f instructions %10.0f branchmisses %10.0f "
+           "cacheref %10.0f cachemisses %10.0f  bytespercachemiss %10.0f "
+           "inspercycle %10.1f insperbyte %10.1f\n",
+           stats[0] * 1.0 / repeat, stats[1] * 1.0 / repeat,
+           stats[2] * 1.0 / repeat, stats[3] * 1.0 / repeat,
+           stats[4] * 1.0 / repeat, volume * repeat * 1.0 / stats[2],
+           stats[1] * 1.0 / stats[0], stats[1] * 1.0 / (volume * repeat));
+
+    std::fill(stats.begin(), stats.end(), 0); // unnecessary
+    for (int i = 0; i < repeat; i++) {
+      memcpy(buffer, p.data(), p.size());
+      unified.start();
+      if (sajson::parse(sajson::bounded_allocation(ast_buffer, ast_buffer_size),
+                        sajson::mutable_string_view(p.size(), buffer))
+              .is_valid() != true)
+        printf("bug\n");
+      unified.end(results);
+      std::transform(stats.begin(), stats.end(), results.begin(), stats.begin(),
+                     std::plus<unsigned long long>());
+    }
+    printf("sajson   : cycles %10.0f instructions %10.0f branchmisses %10.0f "
+           "cacheref %10.0f cachemisses %10.0f  bytespercachemiss %10.0f "
+           "inspercycle %10.1f insperbyte %10.1f\n",
+           stats[0] * 1.0 / repeat, stats[1] * 1.0 / repeat,
+           stats[2] * 1.0 / repeat, stats[3] * 1.0 / repeat,
+           stats[4] * 1.0 / repeat, volume * repeat * 1.0 / stats[2],
+           stats[1] * 1.0 / stats[0], stats[1] * 1.0 / (volume * repeat));
+  }
+#endif //  __linux__
+
   free(ast_buffer);
   free(buffer);
 }

benchmark/statisticalmodel.cpp

@@ -1,5 +1,5 @@
 #include <iostream>
-#ifndef _MSC_VER 
+#ifndef _MSC_VER
 #include <unistd.h>
 #endif
 #include "simdjson/jsonioutil.h"
@@ -8,8 +8,6 @@
 #include "linux-perf-events.h"
 #endif
 
-using namespace std;
-
 size_t count_nonasciibytes(const uint8_t *input, size_t length) {
   size_t count = 0;
   for (size_t i = 0; i < length; i++) {
@@ -31,7 +29,7 @@ struct stat_s {
   size_t float_count;
   size_t string_count;
   size_t backslash_count;
-  size_t nonasciibyte_count;
+  size_t non_ascii_byte_count;
   size_t object_count;
   size_t array_count;
   size_t null_count;
@@ -44,16 +42,17 @@ struct stat_s {
 
 using stat_t = struct stat_s;
 
-stat_t simdjson_computestats(const std::string_view &p) {
+stat_t simdjson_compute_stats(const simdjson::padded_string &p) {
   stat_t answer;
-  ParsedJson pj = build_parsed_json(p);
-  answer.valid = pj.isValid();
+  simdjson::ParsedJson pj = simdjson::build_parsed_json(p);
+  answer.valid = pj.is_valid();
   if (!answer.valid) {
     return answer;
   }
-  answer.backslash_count = count_backslash(reinterpret_cast<const uint8_t *>(p.data()), p.size());
-  answer.nonasciibyte_count =
-      count_nonasciibytes(reinterpret_cast<const uint8_t *>(p.data()), p.size());
+  answer.backslash_count =
+      count_backslash(reinterpret_cast<const uint8_t *>(p.data()), p.size());
+  answer.non_ascii_byte_count = count_nonasciibytes(
+      reinterpret_cast<const uint8_t *>(p.data()), p.size());
   answer.byte_count = p.size();
   answer.integer_count = 0;
   answer.float_count = 0;
@@ -64,24 +63,24 @@ stat_t simdjson_computestats(const std::string_view &p) {
   answer.false_count = 0;
   answer.string_count = 0;
   answer.structural_indexes_count = pj.n_structural_indexes;
-  size_t tapeidx = 0;
-  uint64_t tape_val = pj.tape[tapeidx++];
+  size_t tape_idx = 0;
+  uint64_t tape_val = pj.tape[tape_idx++];
   uint8_t type = (tape_val >> 56);
-  size_t howmany = 0;
+  size_t how_many = 0;
   assert(type == 'r');
-  howmany = tape_val & JSONVALUEMASK;
-  for (; tapeidx < howmany; tapeidx++) {
-    tape_val = pj.tape[tapeidx];
-    // uint64_t payload = tape_val & JSONVALUEMASK;
+  how_many = tape_val & JSON_VALUE_MASK;
+  for (; tape_idx < how_many; tape_idx++) {
+    tape_val = pj.tape[tape_idx];
+    // uint64_t payload = tape_val & JSON_VALUE_MASK;
     type = (tape_val >> 56);
     switch (type) {
     case 'l': // we have a long int
       answer.integer_count++;
-      tapeidx++; // skipping the integer
+      tape_idx++; // skipping the integer
       break;
     case 'd': // we have a double
       answer.float_count++;
-      tapeidx++; // skipping the double
+      tape_idx++; // skipping the double
       break;
     case 'n': // we have a null
       answer.null_count++;
@@ -114,20 +113,20 @@ stat_t simdjson_computestats(const std::string_view &p) {
 
 int main(int argc, char *argv[]) {
 #ifndef _MSC_VER
-	int c;
-	while ((c = getopt(argc, argv, "")) != -1) {
+  int c;
+  while ((c = getopt(argc, argv, "")) != -1) {
     switch (c) {
 
     default:
       abort();
     }
-}
+  }
 #else
   int optind = 1;
 #endif
   if (optind >= argc) {
-    cerr << "Reads json, prints stats. " << endl;
-    cerr << "Usage: " << argv[0] << " <jsonfile>" << endl;
+    std::cerr << "Reads json, prints stats. " << std::endl;
+    std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
 
     exit(1);
   }
@@ -136,65 +135,68 @@ int main(int argc, char *argv[]) {
     std::cerr << "warning: ignoring everything after " << argv[optind + 1]
               << std::endl;
   }
-  std::string_view p;
+  simdjson::padded_string p;
   try {
-    p = get_corpus(filename);
-  } catch (const std::exception &e) { // caught by reference to base
+    simdjson::get_corpus(filename).swap(p);
+  } catch (const std::exception &) { // caught by reference to base
     std::cerr << "Could not load the file " << filename << std::endl;
     return EXIT_FAILURE;
   }
-  stat_t s = simdjson_computestats(p);
+  stat_t s = simdjson_compute_stats(p);
   if (!s.valid) {
     std::cerr << "not a valid JSON" << std::endl;
     return EXIT_FAILURE;
   }
 
   printf("# integer_count float_count string_count backslash_count "
-         "nonasciibyte_count object_count array_count null_count true_count "
+         "non_ascii_byte_count object_count array_count null_count true_count "
          "false_count byte_count structural_indexes_count ");
 #ifdef __linux__
-  printf(
-      "  stage1_cycle_count stage1_instruction_count  stage2_cycle_count "
-      " stage2_instruction_count  stage3_cycle_count stage3_instruction_count  ");
+  printf("  stage1_cycle_count stage1_instruction_count  stage2_cycle_count "
+         " stage2_instruction_count  stage3_cycle_count "
+         "stage3_instruction_count  ");
 #else
   printf("(you are not under linux, so perf counters are disaabled)");
 #endif
   printf("\n");
   printf("%zu %zu %zu %zu %zu %zu %zu %zu %zu %zu %zu %zu ", s.integer_count,
-         s.float_count, s.string_count, s.backslash_count, s.nonasciibyte_count,
-         s.object_count, s.array_count, s.null_count, s.true_count,
-         s.false_count, s.byte_count, s.structural_indexes_count);
+         s.float_count, s.string_count, s.backslash_count,
+         s.non_ascii_byte_count, s.object_count, s.array_count, s.null_count,
+         s.true_count, s.false_count, s.byte_count, s.structural_indexes_count);
 #ifdef __linux__
-  ParsedJson pj;
-  bool allocok = pj.allocateCapacity(p.size());
+  simdjson::ParsedJson pj;
+  bool allocok = pj.allocate_capacity(p.size());
   if (!allocok) {
     std::cerr << "failed to allocate memory" << std::endl;
     return EXIT_FAILURE;
   }
   const uint32_t iterations = p.size() < 1 * 1000 * 1000 ? 1000 : 50;
-  vector<int> evts;
+  std::vector<int> evts;
   evts.push_back(PERF_COUNT_HW_CPU_CYCLES);
   evts.push_back(PERF_COUNT_HW_INSTRUCTIONS);
   LinuxEvents<PERF_TYPE_HARDWARE> unified(evts);
   unsigned long cy1 = 0, cy2 = 0;
   unsigned long cl1 = 0, cl2 = 0;
-  vector<unsigned long long> results;
+  std::vector<unsigned long long> results;
   results.resize(evts.size());
   for (uint32_t i = 0; i < iterations; i++) {
     unified.start();
-    bool isok = find_structural_bits(p.data(), p.size(), pj);
+    // The default template is simdjson::Architecture::NATIVE.
+    bool isok = (simdjson::find_structural_bits<>(p.data(), p.size(), pj) ==
+                 simdjson::SUCCESS);
     unified.end(results);
-    
+
     cy1 += results[0];
     cl1 += results[1];
-    
+
     unified.start();
-    isok = isok && unified_machine(p.data(), p.size(), pj);
+    isok =
+        isok && (simdjson::SUCCESS == unified_machine(p.data(), p.size(), pj));
     unified.end(results);
-    
+
     cy2 += results[0];
     cl2 += results[1];
-    if(!isok) {
+    if (!isok) {
       std::cerr << "failure?" << std::endl;
     }
   }

include/simdjson/common_defs.h

@@ -9,7 +9,13 @@
 #define SIMDJSON_MAXSIZE_BYTES 0xFFFFFFFF
 
 // the input buf should be readable up to buf + SIMDJSON_PADDING
-#define SIMDJSON_PADDING  sizeof(__m256i)
+#ifdef __AVX2__
+#define SIMDJSON_PADDING sizeof(__m256i)
+#else
+// this is a stopgap; there should be a better description of the
+// main loop and its behavior that abstracts over this
+#define SIMDJSON_PADDING 32
+#endif
 
 #ifndef _MSC_VER
 // Implemented using Labels as Values which works in GCC and CLANG (and maybe
@@ -17,9 +23,6 @@
 #define SIMDJSON_USE_COMPUTED_GOTO
 #endif
 
-
-
-
 // Align to N-byte boundary
 #define ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1))
 #define ROUNDDOWN_N(a, n) ((a) & ~((n)-1))
@@ -27,9 +30,7 @@
 #define ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0)
 
 #ifdef _MSC_VER
-
-
-#define really_inline inline
+#define really_inline __forceinline
 #define never_inline __declspec(noinline)
 
 #define UNUSED
@@ -42,8 +43,39 @@
 #define unlikely(x) x
 #endif
 
+// For Visual Studio compilers, same-page buffer overrun is not fine.
+#define ALLOW_SAME_PAGE_BUFFER_OVERRUN false
+
 #else
 
+// For non-Visual Studio compilers, we may assume that same-page buffer overrun
+// is fine. However, it will make it difficult to be "valgrind clean".
+//#ifndef ALLOW_SAME_PAGE_BUFFER_OVERRUN
+//#define ALLOW_SAME_PAGE_BUFFER_OVERRUN true
+//#else
+#define ALLOW_SAME_PAGE_BUFFER_OVERRUN false
+//#endif
+
+// The following is likely unnecessarily complex.
+#ifdef __SANITIZE_ADDRESS__
+// we have GCC, stuck with https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67368
+#define ALLOW_SAME_PAGE_BUFFER_OVERRUN false
+#elif defined(__has_feature)
+// we have CLANG?
+// todo: if we're setting ALLOW_SAME_PAGE_BUFFER_OVERRUN to false, why do we
+// have a non-empty qualifier?
+#if (__has_feature(address_sanitizer))
+#define ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER                               \
+  __attribute__((no_sanitize("address")))
+#endif
+#endif
+
+#if defined(__has_feature)
+#if (__has_feature(memory_sanitizer))
+#define LENIENT_MEM_SANITIZER __attribute__((no_sanitize("memory")))
+#endif
+#endif
+
 #define really_inline inline __attribute__((always_inline, unused))
 #define never_inline inline __attribute__((noinline, unused))
 
@@ -57,6 +89,14 @@
 #define unlikely(x) __builtin_expect(!!(x), 0)
 #endif
 
-#endif  // MSC_VER
+#endif // MSC_VER
+
+// if it does not apply, make it an empty macro
+#ifndef ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER
+#define ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER
+#endif
+#ifndef LENIENT_MEM_SANITIZER
+#define LENIENT_MEM_SANITIZER
+#endif
 
 #endif // SIMDJSON_COMMON_DEFS_H

include/simdjson/isadetection.h

@@ -0,0 +1,152 @@
+/* From
+https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h
+Highly modified.
+
+Copyright (c) 2016-     Facebook, Inc            (Adam Paszke)
+Copyright (c) 2014-     Facebook, Inc            (Soumith Chintala)
+Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)
+Copyright (c) 2012-2014 Deepmind Technologies    (Koray Kavukcuoglu)
+Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)
+Copyright (c) 2011-2013 NYU                      (Clement Farabet)
+Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou,
+Iain Melvin, Jason Weston) Copyright (c) 2006      Idiap Research Institute
+(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert,
+Samy Bengio, Johnny Mariethoz)
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories
+America and IDIAP Research Institute nor the names of its contributors may be
+   used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef SIMDJSON_ISADETECTION_H
+#define SIMDJSON_ISADETECTION_H
+
+#include <stdint.h>
+#include <stdlib.h>
+#if defined(_MSC_VER)
+#include <intrin.h>
+#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)
+#include <cpuid.h>
+#endif
+
+namespace simdjson {
+// Can be found on Intel ISA Reference for CPUID
+constexpr uint32_t cpuid_avx2_bit = 1 << 5;      // Bit 5 of EBX for EAX=0x7
+constexpr uint32_t cpuid_bmi1_bit = 1 << 3;      // bit 3 of EBX for EAX=0x7
+constexpr uint32_t cpuid_bmi2_bit = 1 << 8;      // bit 8 of EBX for EAX=0x7
+constexpr uint32_t cpuid_sse42_bit = 1 << 20;    // bit 20 of ECX for EAX=0x1
+constexpr uint32_t cpuid_pclmulqdq_bit = 1 << 1; // bit  1 of ECX for EAX=0x1
+
+enum instruction_set {
+  DEFAULT = 0x0,
+  NEON = 0x1,
+  AVX2 = 0x4,
+  SSE42 = 0x8,
+  PCLMULQDQ = 0x10,
+  BMI1 = 0x20,
+  BMI2 = 0x40
+};
+
+#if defined(__arm__) || defined(__aarch64__) // incl. armel, armhf, arm64
+
+#if defined(__NEON__)
+
+static inline uint32_t detect_supported_architectures() {
+  return instruction_set::NEON;
+}
+
+#else // ARM without NEON
+
+static inline uint32_t detect_supported_architectures() {
+  return instruction_set::DEFAULT;
+}
+
+#endif
+
+#else // x86
+static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx,
+                         uint32_t *edx) {
+#if defined(_MSC_VER)
+  int cpu_info[4];
+  __cpuid(cpu_info, *eax);
+  *eax = cpu_info[0];
+  *ebx = cpu_info[1];
+  *ecx = cpu_info[2];
+  *edx = cpu_info[3];
+#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)
+  uint32_t level = *eax;
+  __get_cpuid(level, eax, ebx, ecx, edx);
+#else
+  uint32_t a = *eax, b, c = *ecx, d;
+  asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d));
+  *eax = a;
+  *ebx = b;
+  *ecx = c;
+  *edx = d;
+#endif
+}
+
+static inline uint32_t detect_supported_architectures() {
+  uint32_t eax, ebx, ecx, edx;
+  uint32_t host_isa = 0x0;
+
+  // ECX for EAX=0x7
+  eax = 0x7;
+  ecx = 0x0;
+  cpuid(&eax, &ebx, &ecx, &edx);
+
+  if (ebx & cpuid_avx2_bit) {
+    host_isa |= instruction_set::AVX2;
+  }
+
+  if (ebx & cpuid_bmi1_bit) {
+    host_isa |= instruction_set::BMI1;
+  }
+
+  if (ebx & cpuid_bmi2_bit) {
+    host_isa |= instruction_set::BMI2;
+  }
+
+  // EBX for EAX=0x1
+  eax = 0x1;
+  cpuid(&eax, &ebx, &ecx, &edx);
+
+  if (ecx & cpuid_sse42_bit) {
+    host_isa |= instruction_set::SSE42;
+  }
+
+  if (ecx & cpuid_pclmulqdq_bit) {
+    host_isa |= instruction_set::PCLMULQDQ;
+  }
+
+  return host_isa;
+}
+
+#endif // end SIMD extension detection code
+} // namespace simdjson
+#endif

include/simdjson/jsoncharutils.h

@@ -4,6 +4,7 @@
 #include "simdjson/common_defs.h"
 #include "simdjson/parsedjson.h"
 
+namespace simdjson {
 // structural chars here are
 // they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c (and NULL)
 // we are also interested in the four whitespace characters
@@ -11,7 +12,7 @@
 
 // these are the chars that can follow a true/false/null or number atom
 // and nothing else
-const uint32_t structural_or_whitespace_negated[256] = {
+const uint32_t structural_or_whitespace_or_null_negated[256] = {
     0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1,
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1,
@@ -28,13 +29,36 @@ const uint32_t structural_or_whitespace_negated[256] = {
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
 
+// return non-zero if not a structural or whitespace char
+// zero otherwise
+really_inline uint32_t is_not_structural_or_whitespace_or_null(uint8_t c) {
+  return structural_or_whitespace_or_null_negated[c];
+}
+
+const uint32_t structural_or_whitespace_negated[256] = {
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1,
+
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1,
+
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
+
 // return non-zero if not a structural or whitespace char
 // zero otherwise
 really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) {
   return structural_or_whitespace_negated[c];
 }
 
-const uint32_t structural_or_whitespace[256] = {
+const uint32_t structural_or_whitespace_or_null[256] = {
     1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -47,38 +71,189 @@ const uint32_t structural_or_whitespace[256] = {
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 
+really_inline uint32_t is_structural_or_whitespace_or_null(uint8_t c) {
+  return structural_or_whitespace_or_null[c];
+}
+
+const uint32_t structural_or_whitespace[256] = {
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
 really_inline uint32_t is_structural_or_whitespace(uint8_t c) {
   return structural_or_whitespace[c];
 }
 
-const signed char digittoval[256] = {
-    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0,  1,  2,  3,  4,  5,  6,  7,  8,
-    9,  -1, -1, -1, -1, -1, -1, -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1,
-    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-    -1, -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-    -1, -1, -1, -1, -1, -1, -1, -1, -1};
-
+const uint32_t digit_to_val32[886] = {
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0x0,        0x1,        0x2,        0x3,        0x4,        0x5,
+    0x6,        0x7,        0x8,        0x9,        0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa,
+    0xb,        0xc,        0xd,        0xe,        0xf,        0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xa,        0xb,        0xc,        0xd,        0xe,
+    0xf,        0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0x0,        0x10,       0x20,       0x30,       0x40,       0x50,
+    0x60,       0x70,       0x80,       0x90,       0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa0,
+    0xb0,       0xc0,       0xd0,       0xe0,       0xf0,       0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xa0,       0xb0,       0xc0,       0xd0,       0xe0,
+    0xf0,       0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0x0,        0x100,      0x200,      0x300,      0x400,      0x500,
+    0x600,      0x700,      0x800,      0x900,      0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa00,
+    0xb00,      0xc00,      0xd00,      0xe00,      0xf00,      0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xa00,      0xb00,      0xc00,      0xd00,      0xe00,
+    0xf00,      0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0x0,        0x1000,     0x2000,     0x3000,     0x4000,     0x5000,
+    0x6000,     0x7000,     0x8000,     0x9000,     0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa000,
+    0xb000,     0xc000,     0xd000,     0xe000,     0xf000,     0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xa000,     0xb000,     0xc000,     0xd000,     0xe000,
+    0xf000,     0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
+    0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF};
 // returns a value with the high 16 bits set if not valid
-// otherwise returns the conversion of the 4 hex digits at src into the bottom 16 bits of the 32-bit
-// return register
-static inline uint32_t hex_to_u32_nocheck(const uint8_t *src) {// strictly speaking, static inline is a C-ism
-  // all these will sign-extend the chars looked up, placing 1-bits into the high 28 bits of every
-  // invalid value. After the shifts, this will *still* result in the outcome that the high 16 bits of any
-  // value with any invalid char will be all 1's. We check for this in the caller.
-  int32_t v1 = digittoval[src[0]];
-  int32_t v2 = digittoval[src[1]];
-  int32_t v3 = digittoval[src[2]];
-  int32_t v4 = digittoval[src[3]];
-  return static_cast<uint32_t>(v1 << 12 | v2 << 8 | v3 << 4 | v4);
+// otherwise returns the conversion of the 4 hex digits at src into the bottom
+// 16 bits of the 32-bit return register
+//
+// see
+// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/
+static inline uint32_t hex_to_u32_nocheck(
+    const uint8_t *src) { // strictly speaking, static inline is a C-ism
+  uint32_t v1 = digit_to_val32[630 + src[0]];
+  uint32_t v2 = digit_to_val32[420 + src[1]];
+  uint32_t v3 = digit_to_val32[210 + src[2]];
+  uint32_t v4 = digit_to_val32[0 + src[3]];
+  return v1 | v2 | v3 | v4;
 }
 
 // given a code point cp, writes to c
@@ -97,19 +272,21 @@ inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) {
   if (cp <= 0x7F) {
     c[0] = cp;
     return 1; // ascii
-  } if (cp <= 0x7FF) {
+  }
+  if (cp <= 0x7FF) {
     c[0] = (cp >> 6) + 192;
     c[1] = (cp & 63) + 128;
     return 2; // universal plane
-  //  Surrogates are treated elsewhere...
-  //} //else if (0xd800 <= cp && cp <= 0xdfff) {
-  //  return 0; // surrogates // could put assert here
+    //  Surrogates are treated elsewhere...
+    //} //else if (0xd800 <= cp && cp <= 0xdfff) {
+    //  return 0; // surrogates // could put assert here
   } else if (cp <= 0xFFFF) {
     c[0] = (cp >> 12) + 224;
     c[1] = ((cp >> 6) & 63) + 128;
     c[2] = (cp & 63) + 128;
     return 3;
-  } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this is not needed
+  } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this
+                               // is not needed
     c[0] = (cp >> 18) + 240;
     c[1] = ((cp >> 12) & 63) + 128;
     c[2] = ((cp >> 6) & 63) + 128;
@@ -119,5 +296,6 @@ inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) {
   // will return 0 when the code point was too large.
   return 0; // bad r
 }
+} // namespace simdjson
 
 #endif

include/simdjson/jsonformatutils.h

@@ -5,44 +5,45 @@
 #include <iomanip>
 #include <iostream>
 
+namespace simdjson {
 // ends with zero char
 static inline void print_with_escapes(const unsigned char *src) {
   while (*src) {
     switch (*src) {
-      case '\b':
-        putchar('\\');
-        putchar('b');
-        break;
-      case '\f':
-        putchar('\\');
-        putchar('f');
-        break;
-      case '\n':
-        putchar('\\');
-        putchar('n');
-        break;
-      case '\r':
-        putchar('\\');
-        putchar('r');
-        break;
-      case '\"':
-        putchar('\\');
-        putchar('"');
-        break;
-      case '\t':
-        putchar('\\');
-        putchar('t');
-        break;
-      case '\\':
-        putchar('\\');
-        putchar('\\');
-        break;
-      default:
-        if (*src <= 0x1F) {
-          printf("\\u%04x", *src);
-        } else {
-          putchar(*src);
-        }
+    case '\b':
+      putchar('\\');
+      putchar('b');
+      break;
+    case '\f':
+      putchar('\\');
+      putchar('f');
+      break;
+    case '\n':
+      putchar('\\');
+      putchar('n');
+      break;
+    case '\r':
+      putchar('\\');
+      putchar('r');
+      break;
+    case '\"':
+      putchar('\\');
+      putchar('"');
+      break;
+    case '\t':
+      putchar('\\');
+      putchar('t');
+      break;
+    case '\\':
+      putchar('\\');
+      putchar('\\');
+      break;
+    default:
+      if (*src <= 0x1F) {
+        printf("\\u%04x", *src);
+      } else {
+        putchar(*src);
+      }
     }
     src++;
   }
@@ -53,43 +54,43 @@ static inline void print_with_escapes(const unsigned char *src,
                                       std::ostream &os) {
   while (*src) {
     switch (*src) {
-      case '\b':
-        os << '\\';
-        os << 'b';
-        break;
-      case '\f':
-        os << '\\';
-        os << 'f';
-        break;
-      case '\n':
-        os << '\\';
-        os << 'n';
-        break;
-      case '\r':
-        os << '\\';
-        os << 'r';
-        break;
-      case '\"':
-        os << '\\';
-        os << '"';
-        break;
-      case '\t':
-        os << '\\';
-        os << 't';
-        break;
-      case '\\':
-        os << '\\';
-        os << '\\';
-        break;
-      default:
-        if (*src <= 0x1F) {
-          std::ios::fmtflags f(os.flags());
-          os << std::hex << std::setw(4) << std::setfill('0')
-             << static_cast<int>(*src);
-          os.flags(f);
-        } else {
-          os << *src;
-        }
+    case '\b':
+      os << '\\';
+      os << 'b';
+      break;
+    case '\f':
+      os << '\\';
+      os << 'f';
+      break;
+    case '\n':
+      os << '\\';
+      os << 'n';
+      break;
+    case '\r':
+      os << '\\';
+      os << 'r';
+      break;
+    case '\"':
+      os << '\\';
+      os << '"';
+      break;
+    case '\t':
+      os << '\\';
+      os << 't';
+      break;
+    case '\\':
+      os << '\\';
+      os << '\\';
+      break;
+    default:
+      if (*src <= 0x1F) {
+        std::ios::fmtflags f(os.flags());
+        os << std::hex << std::setw(4) << std::setfill('0')
+           << static_cast<int>(*src);
+        os.flags(f);
+      } else {
+        os << *src;
+      }
     }
     src++;
   }
@@ -100,40 +101,40 @@ static inline void print_with_escapes(const unsigned char *src, size_t len) {
   const unsigned char *finalsrc = src + len;
   while (src < finalsrc) {
     switch (*src) {
-      case '\b':
-        putchar('\\');
-        putchar('b');
-        break;
-      case '\f':
-        putchar('\\');
-        putchar('f');
-        break;
-      case '\n':
-        putchar('\\');
-        putchar('n');
-        break;
-      case '\r':
-        putchar('\\');
-        putchar('r');
-        break;
-      case '\"':
-        putchar('\\');
-        putchar('"');
-        break;
-      case '\t':
-        putchar('\\');
-        putchar('t');
-        break;
-      case '\\':
-        putchar('\\');
-        putchar('\\');
-        break;
-      default:
-        if (*src <= 0x1F) {
-          printf("\\u%04x", *src);
-        } else {
-          putchar(*src);
-        }
+    case '\b':
+      putchar('\\');
+      putchar('b');
+      break;
+    case '\f':
+      putchar('\\');
+      putchar('f');
+      break;
+    case '\n':
+      putchar('\\');
+      putchar('n');
+      break;
+    case '\r':
+      putchar('\\');
+      putchar('r');
+      break;
+    case '\"':
+      putchar('\\');
+      putchar('"');
+      break;
+    case '\t':
+      putchar('\\');
+      putchar('t');
+      break;
+    case '\\':
+      putchar('\\');
+      putchar('\\');
+      break;
+    default:
+      if (*src <= 0x1F) {
+        printf("\\u%04x", *src);
+      } else {
+        putchar(*src);
+      }
     }
     src++;
   }
@@ -145,43 +146,43 @@ static inline void print_with_escapes(const unsigned char *src,
   const unsigned char *finalsrc = src + len;
   while (src < finalsrc) {
     switch (*src) {
-      case '\b':
-        os << '\\';
-        os << 'b';
-        break;
-      case '\f':
-        os << '\\';
-        os << 'f';
-        break;
-      case '\n':
-        os << '\\';
-        os << 'n';
-        break;
-      case '\r':
-        os << '\\';
-        os << 'r';
-        break;
-      case '\"':
-        os << '\\';
-        os << '"';
-        break;
-      case '\t':
-        os << '\\';
-        os << 't';
-        break;
-      case '\\':
-        os << '\\';
-        os << '\\';
-        break;
-      default:
-        if (*src <= 0x1F) {
-          std::ios::fmtflags f(os.flags());
-          os << std::hex << std::setw(4) << std::setfill('0')
-             << static_cast<int>(*src);
-          os.flags(f);
-        } else {
-          os << *src;
-        }
+    case '\b':
+      os << '\\';
+      os << 'b';
+      break;
+    case '\f':
+      os << '\\';
+      os << 'f';
+      break;
+    case '\n':
+      os << '\\';
+      os << 'n';
+      break;
+    case '\r':
+      os << '\\';
+      os << 'r';
+      break;
+    case '\"':
+      os << '\\';
+      os << '"';
+      break;
+    case '\t':
+      os << '\\';
+      os << 't';
+      break;
+    case '\\':
+      os << '\\';
+      os << '\\';
+      break;
+    default:
+      if (*src <= 0x1F) {
+        std::ios::fmtflags f(os.flags());
+        os << std::hex << std::setw(4) << std::setfill('0')
+           << static_cast<int>(*src);
+        os.flags(f);
+      } else {
+        os << *src;
+      }
     }
     src++;
   }
@@ -195,6 +196,7 @@ static inline void print_with_escapes(const char *src, std::ostream &os,
                                       size_t len) {
   print_with_escapes(reinterpret_cast<const unsigned char *>(src), os, len);
 }
+} // namespace simdjson
 
 #
 #endif

include/simdjson/jsonioutil.h

@@ -8,33 +8,25 @@
 #include <sstream>
 #include <string>
 
+#include "simdjson/padded_string.h"
 
-// low-level function to allocate memory with padding so we can read passed the "length" bytes
-// safely.
-// if you must provide a pointer to some data, create it with this function:
-// length is the max. size in bytes of the string
-// caller is responsible to free the memory (free(...))
-char * allocate_padded_buffer(size_t length);
-
-
-
-
+namespace simdjson {
 
 // load a file in memory...
 // get a corpus; pad out to cache line so we can always use SIMD
 // throws exceptions in case of failure
 // first element of the pair is a string (null terminated)
 // whereas the second element is the length.
-// caller is responsible to free (free((void*)result.data())))
-// 
+// caller is responsible to free (aligned_free((void*)result.data())))
+//
 // throws an exception if the file cannot be opened, use try/catch
 //      try {
 //        p = get_corpus(filename);
-//      } catch (const std::exception& e) { 
-//        free((void*)p.data());//use aligned_free if you plan to use VisualStudio
+//      } catch (const std::exception& e) {
+//        aligned_free((void*)p.data());
 //        std::cout << "Could not load the file " << filename << std::endl;
 //      }
-std::string_view  get_corpus(const std::string& filename);
-
+padded_string get_corpus(const std::string &filename);
+} // namespace simdjson
 
 #endif

include/simdjson/jsonminifier.h

@@ -1,22 +1,30 @@
 #ifndef SIMDJSON_JSONMINIFIER_H
 #define SIMDJSON_JSONMINIFIER_H
 
+#include "simdjson/padded_string.h"
 #include <cstddef>
 #include <cstdint>
+#include <string_view>
+
+namespace simdjson {
 
 // Take input from buf and remove useless whitespace, write it to out; buf and
 // out can be the same pointer. Result is null terminated,
 // return the string length (minus the null termination).
-size_t jsonminify(const uint8_t *buf, size_t len, uint8_t *out);
+// The accelerated version of this function only runs on AVX2 hardware.
+size_t json_minify(const uint8_t *buf, size_t len, uint8_t *out);
 
-
-static inline size_t jsonminify(const char *buf, size_t len, char *out) {
-    return jsonminify(reinterpret_cast<const uint8_t *>(buf), len, reinterpret_cast<uint8_t *>(out));
+static inline size_t json_minify(const char *buf, size_t len, char *out) {
+  return json_minify(reinterpret_cast<const uint8_t *>(buf), len,
+                     reinterpret_cast<uint8_t *>(out));
 }
 
-
-static inline size_t jsonminify(const std::string_view & p, char *out) {
-    return jsonminify(p.data(), p.size(), out);
+static inline size_t json_minify(const std::string_view &p, char *out) {
+  return json_minify(p.data(), p.size(), out);
 }
 
+static inline size_t json_minify(const padded_string &p, char *out) {
+  return json_minify(p.data(), p.size(), out);
+}
+} // namespace simdjson
 #endif

include/simdjson/jsonparser.h

@@ -1,82 +1,250 @@
 #ifndef SIMDJSON_JSONPARSER_H
 #define SIMDJSON_JSONPARSER_H
-
 #include "simdjson/common_defs.h"
 #include "simdjson/jsonioutil.h"
+#include "simdjson/padded_string.h"
 #include "simdjson/parsedjson.h"
+#include "simdjson/simdjson.h"
 #include "simdjson/stage1_find_marks.h"
 #include "simdjson/stage2_build_tape.h"
-#include "simdjson/simdjson.h"
-
-// Parse a document found in buf, need to preallocate ParsedJson.
-// Return 0 on success, an error code from simdjson/simdjson.h otherwise
-// You can also check validit by calling pj.isValid(). The same ParsedJson can be reused for other documents.
-//
-// If reallocifneeded is true (default) then a temporary buffer is created when needed during processing
-// (a copy of the input string is made).
-// The input buf should be readable up to buf + len + SIMDJSON_PADDING if reallocifneeded is false,
-// all bytes at and after buf + len  are ignored (can be garbage).
-WARN_UNUSED
-int json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true);
-
-// Parse a document found in buf, need to preallocate ParsedJson.
-// Return SUCCESS (an integer = 1) in case of a success. You can also check validity
-// by calling pj.isValid(). The same ParsedJson can be reused for other documents.
-//
-// If reallocifneeded is true (default) then a temporary buffer is created when needed during processing
-// (a copy of the input string is made).
-// The input buf should be readable up to buf + len + SIMDJSON_PADDING  if reallocifneeded is false,
-// all bytes at and after buf + len  are ignored (can be garbage).
-WARN_UNUSED
-inline int json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) {
-  return json_parse(reinterpret_cast<const uint8_t *>(buf), len, pj, reallocifneeded);
-}
-
-// Parse a document found in buf, need to preallocate ParsedJson.
-// Return SUCCESS (an integer = 1) in case of a success. You can also check validity
-// by calling pj.isValid(). The same ParsedJson can be reused for other documents.
-//
-// If reallocifneeded is true (default) then a temporary buffer is created when needed during processing
-// (a copy of the input string is made).
-// the input s should be readable up to s.data() + s.size() + SIMDJSON_PADDING  if reallocifneeded is false,
-// all bytes at and after s.data()+s.size() are ignored (can be garbage).
-WARN_UNUSED
-inline int json_parse(const std::string_view &s, ParsedJson &pj, bool reallocifneeded = true) {
-  return json_parse(s.data(), s.size(), pj, reallocifneeded);
-}
-
-
-// Build a ParsedJson object. You can check validity
-// by calling pj.isValid(). This does the memory allocation needed for ParsedJson.
-// If reallocifneeded is true (default) then a temporary buffer is created when needed during processing
-// (a copy of the input string is made).
-//
-// the input buf should be readable up to buf + len + SIMDJSON_PADDING  if reallocifneeded is false,
-// all bytes at and after buf + len  are ignored (can be garbage).
-WARN_UNUSED
-ParsedJson build_parsed_json(const uint8_t *buf, size_t len, bool reallocifneeded = true);
-
-WARN_UNUSED
-// Build a ParsedJson object. You can check validity
-// by calling pj.isValid(). This does the memory allocation needed for ParsedJson.
-// If reallocifneeded is true (default) then a temporary buffer is created when needed during processing
-// (a copy of the input string is made).
-// The input buf should be readable up to buf + len + SIMDJSON_PADDING if reallocifneeded is false,
-// all bytes at and after buf + len  are ignored (can be garbage).
-inline ParsedJson build_parsed_json(const char * buf, size_t len, bool reallocifneeded = true) {
-  return build_parsed_json(reinterpret_cast<const uint8_t *>(buf), len, reallocifneeded);
-}
-
-// convenience function
-WARN_UNUSED
-// Build a ParsedJson object. You can check validity
-// by calling pj.isValid(). This does the memory allocation needed for ParsedJson.
-// If reallocifneeded is true (default) then a temporary buffer is created when needed during processing
-// (a copy of the input string is made).
-// The input s should be readable up to s.data() + s.size() + SIMDJSON_PADDING  if reallocifneeded is false,
-// all bytes at and after s.data()+s.size() are ignored (can be garbage).
-inline ParsedJson build_parsed_json(const std::string_view &s, bool reallocifneeded = true) {
-  return build_parsed_json(s.data(), s.size(), reallocifneeded);
-}
-
+#include <string>
+#ifdef _MSC_VER
+#include <windows.h>
+// must be included after windows.h
+#include <sysinfoapi.h>
+#else
+#include <unistd.h>
+#endif
+
+namespace simdjson {
+// json_parse_implementation is the generic function, it is specialized for
+// various architectures, e.g., as
+// json_parse_implementation<Architecture::HASWELL> or
+// json_parse_implementation<Architecture::ARM64>
+template <Architecture T>
+int json_parse_implementation(const uint8_t *buf, size_t len, ParsedJson &pj,
+                              bool realloc_if_needed = true) {
+  if (pj.byte_capacity < len) {
+    return simdjson::CAPACITY;
+  }
+  bool reallocated = false;
+  if (realloc_if_needed) {
+#if ALLOW_SAME_PAGE_BUFFER_OVERRUN
+// realloc is needed if the end of the memory crosses a page
+#ifdef _MSC_VER
+    SYSTEM_INFO sysInfo;
+    GetSystemInfo(&sysInfo);
+    long page_size = sysInfo.dwPageSize;
+#else
+    long page_size = sysconf(_SC_PAGESIZE);
+#endif
+    //////////////
+    // We want to check that buf + len - 1 and buf + len - 1 + SIMDJSON_PADDING
+    // are in the same page.
+    // That is, we want to check that
+    // (buf + len - 1) / page_size == (buf + len - 1 + SIMDJSON_PADDING) /
+    // page_size That's true if (buf + len - 1) % page_size + SIMDJSON_PADDING <
+    // page_size.
+    ///////////
+    if ((reinterpret_cast<uintptr_t>(buf + len - 1) % page_size) +
+            SIMDJSON_PADDING <
+        static_cast<uintptr_t>(page_size)) {
+#else // SIMDJSON_SAFE_SAME_PAGE_READ_OVERRUN
+    if (true) { // if not SIMDJSON_SAFE_SAME_PAGE_READ_OVERRUN, we always
+                // reallocate
+#endif
+      const uint8_t *tmp_buf = buf;
+      buf = (uint8_t *)allocate_padded_buffer(len);
+      if (buf == NULL)
+        return simdjson::MEMALLOC;
+      memcpy((void *)buf, tmp_buf, len);
+      reallocated = true;
+    } // if (true) OR if ( (reinterpret_cast<uintptr_t>(buf + len - 1) %
+      // page_size ) + SIMDJSON_PADDING < static_cast<uintptr_t>(page_size) ) {
+  }   // if(realloc_if_needed) {
+  int stage1_is_ok = simdjson::find_structural_bits<T>(buf, len, pj);
+  if (stage1_is_ok != simdjson::SUCCESS) {
+    pj.error_code = stage1_is_ok;
+    return pj.error_code;
+  }
+  int res = unified_machine<T>(buf, len, pj);
+  if (reallocated) {
+    aligned_free((void *)buf);
+  }
+  return res;
+}
+
+// Parse a document found in buf.
+//
+// The content should be a valid JSON document encoded as UTF-8. If there is a
+// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
+// discouraged.
+//
+// You need to preallocate ParsedJson with a capacity of len (e.g.,
+// pj.allocate_capacity(len)).
+//
+// The function returns simdjson::SUCCESS (an integer = 0) in case of a success
+// or an error code from simdjson/simdjson.h in case of failure such as
+// simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth;
+// the simdjson::error_message function converts these error codes into a
+// string).
+//
+// You can also check validity by calling pj.is_valid(). The same ParsedJson can
+// be reused for other documents.
+//
+// If realloc_if_needed is true (default) then a temporary buffer is created
+// when needed during processing (a copy of the input string is made). The input
+// buf should be readable up to buf + len + SIMDJSON_PADDING if
+// realloc_if_needed is false, all bytes at and after buf + len  are ignored
+// (can be garbage). The ParsedJson object can be reused.
+
+int json_parse(const uint8_t *buf, size_t len, ParsedJson &pj,
+               bool realloc_if_needed = true);
+
+// Parse a document found in buf.
+//
+// The content should be a valid JSON document encoded as UTF-8. If there is a
+// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
+// discouraged.
+//
+// You need to preallocate ParsedJson with a capacity of len (e.g.,
+// pj.allocate_capacity(len)).
+//
+// The function returns simdjson::SUCCESS (an integer = 0) in case of a success
+// or an error code from simdjson/simdjson.h in case of failure such as
+// simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth;
+// the simdjson::error_message function converts these error codes into a
+// string).
+//
+// You can also check validity
+// by calling pj.is_valid(). The same ParsedJson can be reused for other
+// documents.
+//
+// If realloc_if_needed is true (default) then a temporary buffer is created
+// when needed during processing (a copy of the input string is made). The input
+// buf should be readable up to buf + len + SIMDJSON_PADDING  if
+// realloc_if_needed is false, all bytes at and after buf + len  are ignored
+// (can be garbage). The ParsedJson object can be reused.
+int json_parse(const char *buf, size_t len, ParsedJson &pj,
+               bool realloc_if_needed = true);
+
+// We do not want to allow implicit conversion from C string to std::string.
+int json_parse(const char *buf, ParsedJson &pj) = delete;
+
+// Parse a document found in in string s.
+// You need to preallocate ParsedJson with a capacity of len (e.g.,
+// pj.allocate_capacity(len)).
+//
+// The function returns simdjson::SUCCESS (an integer = 0) in case of a success
+// or an error code from simdjson/simdjson.h in case of failure such as
+// simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth;
+// the simdjson::error_message function converts these error codes into a
+// string).
+//
+// A temporary buffer is created when needed during processing
+// (a copy of the input string is made).
+inline int json_parse(const std::string &s, ParsedJson &pj) {
+  return json_parse(s.data(), s.length(), pj, true);
+}
+
+// Parse a document found in in string s.
+//
+// The content should be a valid JSON document encoded as UTF-8. If there is a
+// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
+// discouraged.
+//
+// You need to preallocate ParsedJson with a capacity of len (e.g.,
+// pj.allocate_capacity(len)).
+//
+// The function returns simdjson::SUCCESS (an integer = 0) in case of a success
+// or an error code from simdjson/simdjson.h in case of failure such as
+// simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth;
+// the simdjson::error_message function converts these error codes into a
+// string).
+//
+// You can also check validity
+// by calling pj.is_valid(). The same ParsedJson can be reused for other
+// documents.
+inline int json_parse(const padded_string &s, ParsedJson &pj) {
+  return json_parse(s.data(), s.length(), pj, false);
+}
+
+// Build a ParsedJson object. You can check validity
+// by calling pj.is_valid(). This does the memory allocation needed for
+// ParsedJson. If realloc_if_needed is true (default) then a temporary buffer is
+// created when needed during processing (a copy of the input string is made).
+//
+// The input buf should be readable up to buf + len + SIMDJSON_PADDING  if
+// realloc_if_needed is false, all bytes at and after buf + len  are ignored
+// (can be garbage).
+//
+// The content should be a valid JSON document encoded as UTF-8. If there is a
+// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
+// discouraged.
+//
+// This is a convenience function which calls json_parse.
+WARN_UNUSED
+ParsedJson build_parsed_json(const uint8_t *buf, size_t len,
+                             bool realloc_if_needed = true);
+
+WARN_UNUSED
+// Build a ParsedJson object. You can check validity
+// by calling pj.is_valid(). This does the memory allocation needed for
+// ParsedJson. If realloc_if_needed is true (default) then a temporary buffer is
+// created when needed during processing (a copy of the input string is made).
+//
+// The input buf should be readable up to buf + len + SIMDJSON_PADDING if
+// realloc_if_needed is false, all bytes at and after buf + len  are ignored
+// (can be garbage).
+//
+//
+// The content should be a valid JSON document encoded as UTF-8. If there is a
+// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
+// discouraged.
+//
+// This is a convenience function which calls json_parse.
+inline ParsedJson build_parsed_json(const char *buf, size_t len,
+                                    bool realloc_if_needed = true) {
+  return build_parsed_json(reinterpret_cast<const uint8_t *>(buf), len,
+                           realloc_if_needed);
+}
+
+// We do not want to allow implicit conversion from C string to std::string.
+ParsedJson build_parsed_json(const char *buf) = delete;
+
+// Parse a document found in in string s.
+// You need to preallocate ParsedJson with a capacity of len (e.g.,
+// pj.allocate_capacity(len)). Return SUCCESS (an integer = 0) in case of a
+// success. You can also check validity by calling pj.is_valid(). The same
+// ParsedJson can be reused for other documents.
+//
+// A temporary buffer is created when needed during processing
+// (a copy of the input string is made).
+//
+// The content should be a valid JSON document encoded as UTF-8. If there is a
+// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
+// discouraged.
+//
+// This is a convenience function which calls json_parse.
+WARN_UNUSED
+inline ParsedJson build_parsed_json(const std::string &s) {
+  return build_parsed_json(s.data(), s.length(), true);
+}
+
+// Parse a document found in in string s.
+// You need to preallocate ParsedJson with a capacity of len (e.g.,
+// pj.allocate_capacity(len)). Return SUCCESS (an integer = 0) in case of a
+// success. You can also check validity by calling pj.is_valid(). The same
+// ParsedJson can be reused for other documents.
+//
+// The content should be a valid JSON document encoded as UTF-8. If there is a
+// UTF-8 BOM, the caller is responsible for omitting it, UTF-8 BOM are
+// discouraged.
+//
+// This is a convenience function which calls json_parse.
+WARN_UNUSED
+inline ParsedJson build_parsed_json(const padded_string &s) {
+  return build_parsed_json(s.data(), s.length(), false);
+}
+} // namespace simdjson
 #endif

include/simdjson/numberparsing.h

@@ -6,6 +6,18 @@
 #include "simdjson/parsedjson.h"
 #include "simdjson/portability.h"
 
+#ifdef JSON_TEST_NUMBERS // for unit testing
+void found_invalid_number(const uint8_t *buf);
+void found_integer(int64_t result, const uint8_t *buf);
+void found_float(double result, const uint8_t *buf);
+#endif
+
+namespace simdjson {
+// Allowable floating-point values range from
+// std::numeric_limits<double>::lowest() to std::numeric_limits<double>::max(),
+// so from -1.7976e308 all the way to 1.7975e308 in binary64. The lowest
+// non-zero normal values is std::numeric_limits<double>::min() or
+// about 2.225074e-308.
 static const double power_of_ten[] = {
     1e-308, 1e-307, 1e-306, 1e-305, 1e-304, 1e-303, 1e-302, 1e-301, 1e-300,
     1e-299, 1e-298, 1e-297, 1e-296, 1e-295, 1e-294, 1e-293, 1e-292, 1e-291,
@@ -102,17 +114,22 @@ really_inline bool
 is_not_structural_or_whitespace_or_exponent_or_decimal(unsigned char c) {
   return structural_or_whitespace_or_exponent_or_decimal_negated[c];
 }
-
+} // namespace simdjson
+#ifndef SIMDJSON_DISABLE_SWAR_NUMBER_PARSING
 #define SWAR_NUMBER_PARSING
+#endif
 
 #ifdef SWAR_NUMBER_PARSING
 
-#ifdef _MSC_VER
+namespace simdjson {
 // check quickly whether the next 8 chars are made of digits
 // at a glance, it looks better than Mula's
 // http://0x80.pl/articles/swar-digits-validate.html
 static inline bool is_made_of_eight_digits_fast(const char *chars) {
   uint64_t val;
+  // this can read up to 7 bytes beyond the buffer size, but we require
+  // SIMDJSON_PADDING of padding
+  static_assert(7 <= SIMDJSON_PADDING);
   memcpy(&val, chars, 8);
   // a branchy method might be faster:
   // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030)
@@ -122,17 +139,10 @@ static inline bool is_made_of_eight_digits_fast(const char *chars) {
            (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) ==
           0x3333333333333333);
 }
-#else
-// this is more efficient apparently than the scalar code above (fewer instructions)
-static inline bool is_made_of_eight_digits_fast(const char *chars) {
-  __m64 val;
-  memcpy(&val, chars, 8);
-  __m64 base = _mm_sub_pi8(val,_mm_set1_pi8('0'));
-  __m64 basecmp = _mm_subs_pu8(base,_mm_set1_pi8(9));
-  return _mm_cvtm64_si64(basecmp) == 0;
-}
-#endif
-
+} // namespace simdjson
+#ifdef IS_X86_64
+TARGET_WESTMERE
+namespace simdjson {
 static inline uint32_t parse_eight_digits_unrolled(const char *chars) {
   // this actually computes *16* values so we are being wasteful.
   const __m128i ascii0 = _mm_set1_epi8('0');
@@ -141,7 +151,8 @@ static inline uint32_t parse_eight_digits_unrolled(const char *chars) {
   const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1);
   const __m128i mul_1_10000 =
       _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1);
-  const __m128i input = _mm_sub_epi8(_mm_loadu_si128(reinterpret_cast<const __m128i *>(chars)), ascii0);
+  const __m128i input = _mm_sub_epi8(
+      _mm_loadu_si128(reinterpret_cast<const __m128i *>(chars)), ascii0);
   const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10);
   const __m128i t2 = _mm_madd_epi16(t1, mul_1_100);
   const __m128i t3 = _mm_packus_epi32(t2, t2);
@@ -149,9 +160,33 @@ static inline uint32_t parse_eight_digits_unrolled(const char *chars) {
   return _mm_cvtsi128_si32(
       t4); // only captures the sum of the first 8 digits, drop the rest
 }
+} // namespace simdjson
+UNTARGET_REGION
+#endif
+
+namespace simdjson {
+#ifdef IS_ARM64
+// we don't have SSE, so let us use a scalar function
+// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
+static inline uint32_t parse_eight_digits_unrolled(const char *chars) {
+  uint64_t val;
+  memcpy(&val, chars, sizeof(uint64_t));
+  val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8;
+  val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16;
+  return (val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32;
+}
+#endif
 
 #endif
 
+//
+// This function computes base * 10 ^ (- negative_exponent ).
+// It is only even going to be used when negative_exponent is tiny.
+static double subnormal_power10(double base, int negative_exponent) {
+  // this is probably not going to be fast
+  return base * 1e-308 * pow(10, negative_exponent + 308);
+}
+
 // called by parse_number when we know that the output is a float,
 // but where there might be some integer overflow. The trick here is to
 // parse using floats from the start.
@@ -162,17 +197,15 @@ static inline uint32_t parse_eight_digits_unrolled(const char *chars) {
 //
 // Note: a redesign could avoid this function entirely.
 //
-static never_inline bool
-parse_float(const uint8_t *const buf,
-                          ParsedJson &pj, const uint32_t offset,
-                          bool found_minus) {
+static never_inline bool parse_float(const uint8_t *const buf, ParsedJson &pj,
+                                     const uint32_t offset, bool found_minus) {
   const char *p = reinterpret_cast<const char *>(buf + offset);
   bool negative = false;
   if (found_minus) {
     ++p;
     negative = true;
   }
-  double i;
+  long double i;
   if (*p == '0') { // 0 cannot be followed by an integer
     ++p;
     i = 0;
@@ -188,82 +221,102 @@ parse_float(const uint8_t *const buf,
   }
   if ('.' == *p) {
     ++p;
-    double fractionalweight = 1;
-    if(is_integer(*p)) {
+    int fractional_weight = 308;
+    if (is_integer(*p)) {
       unsigned char digit = *p - '0';
       ++p;
-      fractionalweight *= 0.1;
-      i = i + digit * fractionalweight;
+
+      fractional_weight--;
+      i = i + digit * (fractional_weight >= 0 ? power_of_ten[fractional_weight]
+                                              : 0);
     } else {
 #ifdef JSON_TEST_NUMBERS // for unit testing
-      foundInvalidNumber(buf + offset);
+      found_invalid_number(buf + offset);
 #endif
       return false;
     }
     while (is_integer(*p)) {
       unsigned char digit = *p - '0';
       ++p;
-      fractionalweight *= 0.1;
-      i = i + digit * fractionalweight;
+      fractional_weight--;
+      i = i + digit * (fractional_weight >= 0 ? power_of_ten[fractional_weight]
+                                              : 0);
     }
   }
   if (('e' == *p) || ('E' == *p)) {
     ++p;
-    bool negexp = false;
+    bool neg_exp = false;
     if ('-' == *p) {
-      negexp = true;
+      neg_exp = true;
       ++p;
     } else if ('+' == *p) {
       ++p;
     }
     if (!is_integer(*p)) {
 #ifdef JSON_TEST_NUMBERS // for unit testing
-      foundInvalidNumber(buf + offset);
+      found_invalid_number(buf + offset);
 #endif
       return false;
     }
     unsigned char digit = *p - '0';
-    int64_t expnumber = digit; // exponential part
+    int64_t exp_number = digit; // exponential part
     p++;
     if (is_integer(*p)) {
       digit = *p - '0';
-      expnumber = 10 * expnumber + digit;
+      exp_number = 10 * exp_number + digit;
       ++p;
     }
     if (is_integer(*p)) {
       digit = *p - '0';
-      expnumber = 10 * expnumber + digit;
+      exp_number = 10 * exp_number + digit;
       ++p;
     }
     if (is_integer(*p)) {
       digit = *p - '0';
-      expnumber = 10 * expnumber + digit;
+      exp_number = 10 * exp_number + digit;
       ++p;
     }
-    if (is_integer(*p)) {
+    while (is_integer(*p)) {
+      if (exp_number > 0x100000000) { // we need to check for overflows
 // we refuse to parse this
 #ifdef JSON_TEST_NUMBERS // for unit testing
-      foundInvalidNumber(buf + offset);
+        found_invalid_number(buf + offset);
 #endif
-      return false;
+        return false;
+      }
+      digit = *p - '0';
+      exp_number = 10 * exp_number + digit;
+      ++p;
     }
-    int exponent = (negexp ? -expnumber : expnumber);
-    if ((exponent > 308) || (exponent < -308)) {
+    if (unlikely(exp_number > 308)) {
+      // this path is unlikely
+      if (neg_exp) {
+        // We either have zero or a subnormal.
+        // We expect this to be uncommon so we go through a slow path.
+        i = subnormal_power10(i, -exp_number);
+      } else {
+// We know for sure that we have a number that is too large,
 // we refuse to parse this
 #ifdef JSON_TEST_NUMBERS // for unit testing
-      foundInvalidNumber(buf + offset);
+        found_invalid_number(buf + offset);
 #endif
-      return false;
+        return false;
+      }
+    } else {
+      int exponent = (neg_exp ? -exp_number : exp_number);
+      // we have that exp_number is [0,308] so that
+      // exponent is [-308,308] so that
+      // 308 + exponent is in [0, 2 * 308]
+      i *= power_of_ten[308 + exponent];
     }
-    i *= power_of_ten[308 + exponent];
   }
-  if(is_not_structural_or_whitespace(*p)) {
+  if (is_not_structural_or_whitespace(*p)) {
     return false;
   }
   double d = negative ? -i : i;
   pj.write_tape_double(d);
 #ifdef JSON_TEST_NUMBERS // for unit testing
-  foundFloat(d, buf + offset);
+  found_float(d, buf + offset);
 #endif
   return is_structural_or_whitespace(*p);
 }
@@ -301,13 +354,13 @@ static never_inline bool parse_large_integer(const uint8_t *const buf,
       digit = *p - '0';
       if (mul_overflow(i, 10, &i)) {
 #ifdef JSON_TEST_NUMBERS // for unit testing
-        foundInvalidNumber(buf + offset);
+        found_invalid_number(buf + offset);
 #endif
         return false; // overflow
       }
       if (add_overflow(i, digit, &i)) {
 #ifdef JSON_TEST_NUMBERS // for unit testing
-        foundInvalidNumber(buf + offset);
+        found_invalid_number(buf + offset);
 #endif
         return false; // overflow
       }
@@ -318,7 +371,7 @@ static never_inline bool parse_large_integer(const uint8_t *const buf,
     if (i > 0x8000000000000000) {
 // overflows!
 #ifdef JSON_TEST_NUMBERS // for unit testing
-      foundInvalidNumber(buf + offset);
+      found_invalid_number(buf + offset);
 #endif
       return false; // overflow
     }
@@ -326,31 +379,36 @@ static never_inline bool parse_large_integer(const uint8_t *const buf,
     if (i >= 0x8000000000000000) {
 // overflows!
 #ifdef JSON_TEST_NUMBERS // for unit testing
-      foundInvalidNumber(buf + offset);
+      found_invalid_number(buf + offset);
 #endif
       return false; // overflow
     }
   }
-  int64_t signed_answer = negative ? -static_cast<int64_t>(i) : static_cast<int64_t>(i);
+  int64_t signed_answer =
+      negative ? -static_cast<int64_t>(i) : static_cast<int64_t>(i);
   pj.write_tape_s64(signed_answer);
 #ifdef JSON_TEST_NUMBERS // for unit testing
-  foundInteger(signed_answer, buf + offset);
+  found_integer(signed_answer, buf + offset);
 #endif
   return is_structural_or_whitespace(*p);
 }
 
-
-
-
 // parse the number at buf + offset
 // define JSON_TEST_NUMBERS for unit testing
-static really_inline bool parse_number(const uint8_t *const buf,
-                                       ParsedJson &pj,
+//
+// It is assumed that the number is followed by a structural ({,},],[) character
+// or a white space character. If that is not the case (e.g., when the JSON
+// document is made of a single number), then it is necessary to copy the
+// content and append a space before calling this function.
+//
+// Our objective is accurate parsing (ULP of 0 or 1) at high speed.
+static really_inline bool parse_number(const uint8_t *const buf, ParsedJson &pj,
                                        const uint32_t offset,
                                        bool found_minus) {
-#ifdef SIMDJSON_SKIPNUMBERPARSING // for performance analysis, it is sometimes useful to skip parsing
-  pj.write_tape_s64(0); // always write zero
-  return true; // always succeeds
+#ifdef SIMDJSON_SKIPNUMBERPARSING // for performance analysis, it is sometimes
+                                  // useful to skip parsing
+  pj.write_tape_s64(0);           // always write zero
+  return true;                    // always succeeds
 #else
   const char *p = reinterpret_cast<const char *>(buf + offset);
   bool negative = false;
@@ -358,28 +416,28 @@ static really_inline bool parse_number(const uint8_t *const buf,
     ++p;
     negative = true;
     if (!is_integer(*p)) { // a negative sign must be followed by an integer
-#ifdef JSON_TEST_NUMBERS   // for unit testing
-      foundInvalidNumber(buf + offset);
+#ifdef JSON_TEST_NUMBERS // for unit testing
+      found_invalid_number(buf + offset);
 #endif
       return false;
     }
   }
-  const char *const startdigits = p;
+  const char *const start_digits = p;
 
-  int64_t i;
+  uint64_t i;      // an unsigned int avoids signed overflows (which are bad)
   if (*p == '0') { // 0 cannot be followed by an integer
     ++p;
     if (is_not_structural_or_whitespace_or_exponent_or_decimal(*p)) {
 #ifdef JSON_TEST_NUMBERS // for unit testing
-      foundInvalidNumber(buf + offset);
+      found_invalid_number(buf + offset);
 #endif
       return false;
     }
     i = 0;
   } else {
     if (!(is_integer(*p))) { // must start with an integer
-#ifdef JSON_TEST_NUMBERS     // for unit testing
-      foundInvalidNumber(buf + offset);
+#ifdef JSON_TEST_NUMBERS // for unit testing
+      found_invalid_number(buf + offset);
 #endif
       return false;
     }
@@ -390,25 +448,33 @@ static really_inline bool parse_number(const uint8_t *const buf,
     // we rarely see large integer parts like 123456789
     while (is_integer(*p)) {
       digit = *p - '0';
-      i = 10 * i + digit; // might overflow
+      // a multiplication by 10 is cheaper than an arbitrary integer
+      // multiplication
+      i = 10 * i + digit; // might overflow, we will handle the overflow later
       ++p;
     }
   }
-
   int64_t exponent = 0;
-
+  bool is_float = false;
   if ('.' == *p) {
+    is_float = true; // At this point we know that we have a float
+    // we continue with the fiction that we have an integer. If the
+    // floating point number is representable as x * 10^z for some integer
+    // z that fits in 53 bits, then we will be able to convert back the
+    // the integer into a float in a lossless manner.
     ++p;
-    const char *const firstafterperiod = p;
-    if(is_integer(*p)) {
+    const char *const first_after_period = p;
+    if (is_integer(*p)) {
       unsigned char digit = *p - '0';
       ++p;
-      i = i * 10 + digit;
+      i = i * 10 + digit; // might overflow + multiplication by 10 is likely
+                          // cheaper than arbitrary mult.
+      // we will handle the overflow later
     } else {
 #ifdef JSON_TEST_NUMBERS // for unit testing
-      foundInvalidNumber(buf + offset);
+      found_invalid_number(buf + offset);
 #endif
-      return false;      
+      return false;
     }
 #ifdef SWAR_NUMBER_PARSING
     // this helps if we have lots of decimals!
@@ -416,100 +482,106 @@ static really_inline bool parse_number(const uint8_t *const buf,
     if (is_made_of_eight_digits_fast(p)) {
       i = i * 100000000 + parse_eight_digits_unrolled(p);
       p += 8;
-      // exponent -= 8;
     }
 #endif
     while (is_integer(*p)) {
       unsigned char digit = *p - '0';
       ++p;
-      i = i * 10 + digit; // in rare cases, this will overflow, but that's ok because we have parse_highprecision_float later.
+      i = i * 10 + digit; // in rare cases, this will overflow, but that's ok
+                          // because we have parse_highprecision_float later.
     }
-    exponent = firstafterperiod - p;
+    exponent = first_after_period - p;
   }
-  int digitcount = p - startdigits - 1;
-
-  int64_t expnumber = 0; // exponential part
+  int digit_count =
+      p - start_digits - 1; // used later to guard against overflows
+  int64_t exp_number = 0;   // exponential part
   if (('e' == *p) || ('E' == *p)) {
+    is_float = true;
     ++p;
-    bool negexp = false;
+    bool neg_exp = false;
     if ('-' == *p) {
-      negexp = true;
+      neg_exp = true;
       ++p;
     } else if ('+' == *p) {
       ++p;
     }
     if (!is_integer(*p)) {
 #ifdef JSON_TEST_NUMBERS // for unit testing
-      foundInvalidNumber(buf + offset);
+      found_invalid_number(buf + offset);
 #endif
       return false;
     }
     unsigned char digit = *p - '0';
-    expnumber = digit;
+    exp_number = digit;
     p++;
     if (is_integer(*p)) {
       digit = *p - '0';
-      expnumber = 10 * expnumber + digit;
+      exp_number = 10 * exp_number + digit;
       ++p;
     }
     if (is_integer(*p)) {
       digit = *p - '0';
-      expnumber = 10 * expnumber + digit;
+      exp_number = 10 * exp_number + digit;
       ++p;
     }
-    if (is_integer(*p)) {
-// we refuse to parse this
+    while (is_integer(*p)) {
+      if (exp_number > 0x100000000) { // we need to check for overflows
+                                      // we refuse to parse this
 #ifdef JSON_TEST_NUMBERS // for unit testing
-      foundInvalidNumber(buf + offset);
-#endif
-      return false;
-    }
-    exponent += (negexp ? -expnumber : expnumber);
-  }
-  i = negative ? -i : i;
-  if ((exponent != 0) || (expnumber != 0)) {
-    if (unlikely(digitcount >= 19)) { // this is uncommon!!!
-      // this is almost never going to get called!!!
-      // we start anew, going slowly!!!
-      return parse_float(buf, pj, offset,
-                                       found_minus);
-    }
-    ///////////
-    // We want 0.1e1 to be a float.
-    //////////
-    if (i == 0) {
-      pj.write_tape_double(0.0);
-#ifdef JSON_TEST_NUMBERS // for unit testing
-      foundFloat(0.0, buf + offset);
-#endif
-    } else {
-      if ((exponent > 308) || (exponent < -308)) {
-// we refuse to parse this
-#ifdef JSON_TEST_NUMBERS // for unit testing
-        foundInvalidNumber(buf + offset);
+        found_invalid_number(buf + offset);
 #endif
         return false;
       }
-      double d = i;
-      d *= power_of_ten[308 + exponent];
-      // d = negative ? -d : d;
-      pj.write_tape_double(d);
+      digit = *p - '0';
+      exp_number = 10 * exp_number + digit;
+      ++p;
+    }
+    exponent += (neg_exp ? -exp_number : exp_number);
+  }
+  if (is_float) {
+    uint64_t power_index = 308 + exponent;
+    if (unlikely((digit_count >= 19))) { // this is uncommon
+      // It is possible that the integer had an overflow.
+      // We have to handle the case where we have 0.0000somenumber.
+      const char *start = start_digits;
+      while ((*start == '0') || (*start == '.')) {
+        start++;
+      }
+      // we over-decrement by one when there is a '.'
+      digit_count -= (start - start_digits);
+      if (digit_count >= 19) {
+        // Ok, chances are good that we had an overflow!
+        // this is almost never going to get called!!!
+        // we start anew, going slowly!!!
+        return parse_float(buf, pj, offset, found_minus);
+      }
+    }
+    if (unlikely((power_index > 2 * 308))) { // this is uncommon!!!
+      // this is almost never going to get called!!!
+      // we start anew, going slowly!!!
+      return parse_float(buf, pj, offset, found_minus);
+    }
+    double factor = power_of_ten[power_index];
+    factor = negative ? -factor : factor;
+    double d = i * factor;
+    pj.write_tape_double(d);
 #ifdef JSON_TEST_NUMBERS // for unit testing
-      foundFloat(d, buf + offset);
+    found_float(d, buf + offset);
 #endif
-    }
   } else {
-    if (unlikely(digitcount >= 18)) { // this is uncommon!!!
-      return parse_large_integer(buf, pj, offset,
-                                 found_minus);
+    if (unlikely(digit_count >= 18)) { // this is uncommon!!!
+      // there is a good chance that we had an overflow, so we need
+      // need to recover: we parse the whole thing again.
+      return parse_large_integer(buf, pj, offset, found_minus);
     }
+    i = negative ? 0 - i : i;
     pj.write_tape_s64(i);
 #ifdef JSON_TEST_NUMBERS // for unit testing
-    foundInteger(i, buf + offset);
+    found_integer(i, buf + offset);
 #endif
   }
-  return  is_structural_or_whitespace(*p);
+  return is_structural_or_whitespace(*p);
 #endif // SIMDJSON_SKIPNUMBERPARSING
 }
-
+} // simdjson
 #endif

include/simdjson/padded_string.h

@@ -0,0 +1,70 @@
+#ifndef SIMDJSON_PADDING_STRING_H
+#define SIMDJSON_PADDING_STRING_H
+#include "simdjson/portability.h"
+#include <cstring>
+#include <memory>
+
+namespace simdjson {
+// low-level function to allocate memory with padding so we can read passed the
+// "length" bytes safely. if you must provide a pointer to some data, create it
+// with this function: length is the max. size in bytes of the string caller is
+// responsible to free the memory (free(...))
+char *allocate_padded_buffer(size_t length);
+
+// Simple string with padded allocation.
+// We deliberately forbid copies, users should rely on swap or move
+// constructors.
+class padded_string {
+public:
+  explicit padded_string() noexcept : viable_size(0), data_ptr(nullptr) {}
+  explicit padded_string(size_t length) noexcept
+      : viable_size(length), data_ptr(allocate_padded_buffer(length)) {
+
+    if (data_ptr != nullptr)
+      data_ptr[length] = '\0'; // easier when you need a c_str
+  }
+  explicit padded_string(char *data, size_t length) noexcept
+      : viable_size(length), data_ptr(allocate_padded_buffer(length)) {
+    if (data_ptr != nullptr) {
+      memcpy(data_ptr, data, length);
+      data_ptr[length] = '\0'; // easier when you need a c_str
+    }
+  }
+  padded_string(std::string s) noexcept
+      : viable_size(s.size()), data_ptr(allocate_padded_buffer(s.size())) {
+    if (data_ptr != nullptr) {
+      memcpy(data_ptr, s.data(), s.size());
+      data_ptr[s.size()] = '\0'; // easier when you need a c_str
+    }
+  }
+  padded_string(padded_string &&o) noexcept
+      : viable_size(o.viable_size), data_ptr(o.data_ptr) {
+    o.data_ptr = nullptr; // we take ownership
+  }
+  void swap(padded_string &o) {
+    size_t tmp_viable_size = viable_size;
+    char *tmp_data_ptr = data_ptr;
+    viable_size = o.viable_size;
+    data_ptr = o.data_ptr;
+    o.data_ptr = tmp_data_ptr;
+    o.viable_size = tmp_viable_size;
+  }
+
+  ~padded_string() { aligned_free_char(data_ptr); }
+
+  size_t size() const { return viable_size; }
+
+  size_t length() const { return viable_size; }
+
+  char *data() const { return data_ptr; }
+
+private:
+  padded_string &operator=(const padded_string &o) = delete;
+  padded_string(const padded_string &o) = delete;
+
+  size_t viable_size;
+  char *data_ptr;
+};
+} // namespace simdjson
+
+#endif

include/simdjson/parsedjson.h

@@ -1,40 +1,50 @@
 #ifndef SIMDJSON_PARSEDJSON_H
 #define SIMDJSON_PARSEDJSON_H
 
+#include "simdjson/common_defs.h"
+#include "simdjson/jsonformatutils.h"
+#include "simdjson/portability.h"
+#include "simdjson/simdjson.h"
 #include <cinttypes>
 #include <cmath>
 #include <cstring>
 #include <iomanip>
 #include <iostream>
+#include <limits>
 
-#include "simdjson/common_defs.h"
-#include "simdjson/jsonformatutils.h"
-#include "simdjson/portability.h"
-
-#define JSONVALUEMASK 0xFFFFFFFFFFFFFF
-
-#define DEFAULTMAXDEPTH 1024// a JSON document with a depth exceeding 1024 is probably de facto invalid
+#define JSON_VALUE_MASK 0xFFFFFFFFFFFFFF
 
+#define DEFAULT_MAX_DEPTH                                                      \
+  1024 // a JSON document with a depth exceeding 1024 is probably de facto
+       // invalid
 
+namespace simdjson {
 /************
  * The JSON is parsed to a tape, see the accompanying tape.md file
  * for documentation.
  ***********/
-struct ParsedJson {
+class ParsedJson {
 public:
-
-  // create a ParsedJson container with zero capacity, call allocateCapacity to
+  // create a ParsedJson container with zero capacity, call allocate_capacity to
   // allocate memory
   ParsedJson();
   ~ParsedJson();
-  ParsedJson(ParsedJson && p);
+  ParsedJson(ParsedJson &&p);
 
   // if needed, allocate memory so that the object is able to process JSON
-  // documents having up to len bytes and maxdepth "depth"
+  // documents having up to len bytes and max_depth "depth"
   WARN_UNUSED
-  bool allocateCapacity(size_t len, size_t maxdepth = DEFAULTMAXDEPTH);
+  bool allocate_capacity(size_t len, size_t max_depth = DEFAULT_MAX_DEPTH);
 
-  bool isValid() const;
+  // returns true if the document parsed was valid
+  bool is_valid() const;
+
+  // return an error code corresponding to the last parsing attempt, see
+  // simdjson.h will return simdjson::UNITIALIZED if no parsing was attempted
+  int get_error_code() const;
+
+  // return the string equivalent of "get_error_code"
+  std::string get_error_message() const;
 
   // deallocate memory and set capacity to zero, called automatically by the
   // destructor
@@ -47,11 +57,10 @@ public:
   // return false if the tape is likely wrong (e.g., you did not parse a valid
   // JSON).
   WARN_UNUSED
-  bool printjson(std::ostream &os);
+  bool print_json(std::ostream &os);
   WARN_UNUSED
   bool dump_raw_tape(std::ostream &os);
 
-
   // all nodes are stored on the tape using a 64-bit word.
   //
   // strings, double and ints are stored as
@@ -68,154 +77,252 @@ public:
 
   // this should be considered a private function
   really_inline void write_tape(uint64_t val, uint8_t c) {
-      tape[current_loc++] = val | ((static_cast<uint64_t>(c)) << 56);
+    tape[current_loc++] = val | ((static_cast<uint64_t>(c)) << 56);
   }
 
   really_inline void write_tape_s64(int64_t i) {
-      write_tape(0, 'l');
-      tape[current_loc++] = *(reinterpret_cast<uint64_t *>(&i));
+    write_tape(0, 'l');
+    tape[current_loc++] = *(reinterpret_cast<uint64_t *>(&i));
   }
 
   really_inline void write_tape_double(double d) {
     write_tape(0, 'd');
     static_assert(sizeof(d) == sizeof(tape[current_loc]), "mismatch size");
-    memcpy(& tape[current_loc++], &d, sizeof(double));
-    //tape[current_loc++] = *((uint64_t *)&d);
+    memcpy(&tape[current_loc++], &d, sizeof(double));
+    // tape[current_loc++] = *((uint64_t *)&d);
   }
 
   really_inline uint32_t get_current_loc() { return current_loc; }
 
-  really_inline void annotate_previousloc(uint32_t saved_loc, uint64_t val) {
-      tape[saved_loc] |= val;
+  really_inline void annotate_previous_loc(uint32_t saved_loc, uint64_t val) {
+    tape[saved_loc] |= val;
   }
 
-  struct iterator {
-    explicit iterator(ParsedJson &pj_);
-    ~iterator();
+  class InvalidJSON : public std::exception {
+    const char *what() const throw() { return "JSON document is invalid"; }
+  };
 
-    iterator(const iterator &o);
+  class Iterator {
+    // might throw InvalidJSON if ParsedJson is invalid
+  public:
+    explicit Iterator(ParsedJson &pj_);
+    ~Iterator();
 
-    iterator(iterator &&o);
+    Iterator(const Iterator &o) noexcept;
 
-    bool isOk() const;
+    Iterator(Iterator &&o) noexcept;
+
+    inline bool is_ok() const;
 
     // useful for debuging purposes
-    size_t get_tape_location() const;
+    inline size_t get_tape_location() const;
 
     // useful for debuging purposes
-    size_t get_tape_length() const;
+    inline size_t get_tape_length() const;
 
-    // returns the current depth (start at 1 with 0 reserved for the fictitious root node)
-    size_t get_depth() const;
+    // returns the current depth (start at 1 with 0 reserved for the fictitious
+    // root node)
+    inline size_t get_depth() const;
 
-    // A scope is a series of nodes at the same depth, typically it is either an object ({) or an array ([).
-    // The root node has type 'r'.
-    uint8_t get_scope_type() const;
+    // A scope is a series of nodes at the same depth, typically it is either an
+    // object ({) or an array ([). The root node has type 'r'.
+    inline uint8_t get_scope_type() const;
 
     // move forward in document order
-    bool move_forward();
+    inline bool move_forward();
 
     // retrieve the character code of what we're looking at:
     // [{"sltfn are the possibilities
-    uint8_t get_type()  const;
+    inline uint8_t get_type() const {
+      return current_type; // short functions should be inlined!
+    }
 
     // get the int64_t value at this node; valid only if we're at "l"
-    int64_t get_integer()  const;
+    inline int64_t get_integer() const {
+      if (location + 1 >= tape_length) {
+        return 0; // default value in case of error
+      }
+      return static_cast<int64_t>(pj.tape[location + 1]);
+    }
 
     // get the string value at this node (NULL ended); valid only if we're at "
-    // note that tabs, and line endings are escaped in the returned value (see print_with_escapes)
-    // return value is valid UTF-8
-    // It may contain NULL chars within the string: get_string_length determines the true 
-    // string length.
-    const char * get_string() const;
+    // note that tabs, and line endings are escaped in the returned value (see
+    // print_with_escapes) return value is valid UTF-8 It may contain NULL chars
+    // within the string: get_string_length determines the true string length.
+    inline const char *get_string() const {
+      return reinterpret_cast<const char *>(
+          pj.string_buf + (current_val & JSON_VALUE_MASK) + sizeof(uint32_t));
+    }
 
-    uint32_t get_string_length() const;
+    // return the length of the string in bytes
+    inline uint32_t get_string_length() const {
+      uint32_t answer;
+      memcpy(&answer,
+             reinterpret_cast<const char *>(pj.string_buf +
+                                            (current_val & JSON_VALUE_MASK)),
+             sizeof(uint32_t));
+      return answer;
+    }
 
     // get the double value at this node; valid only if
     // we're at "d"
-    double get_double()  const;
+    inline double get_double() const {
+      if (location + 1 >= tape_length) {
+        return std::numeric_limits<double>::quiet_NaN(); // default value in
+                                                         // case of error
+      }
+      double answer;
+      memcpy(&answer, &pj.tape[location + 1], sizeof(answer));
+      return answer;
+    }
 
-    bool is_object_or_array() const;
+    inline bool is_object_or_array() const { return is_object() || is_array(); }
 
-    bool is_object() const;
+    inline bool is_object() const { return get_type() == '{'; }
 
-    bool is_array() const;
+    inline bool is_array() const { return get_type() == '['; }
 
-    bool is_string() const;
+    inline bool is_string() const { return get_type() == '"'; }
 
-    bool is_integer() const;
+    inline bool is_integer() const { return get_type() == 'l'; }
 
-    bool is_double() const;
+    inline bool is_double() const { return get_type() == 'd'; }
 
-    static bool is_object_or_array(uint8_t type);
+    inline bool is_true() const { return get_type() == 't'; }
+
+    inline bool is_false() const { return get_type() == 'f'; }
+
+    inline bool is_null() const { return get_type() == 'n'; }
+
+    static bool is_object_or_array(uint8_t type) {
+      return ((type == '[') || (type == '{'));
+    }
 
     // when at {, go one level deep, looking for a given key
     // if successful, we are left pointing at the value,
     // if not, we are still pointing at the object ({)
-    // (in case of repeated keys, this only finds the first one)
+    // (in case of repeated keys, this only finds the first one).
     // We seek the key using C's strcmp so if your JSON strings contain
     // NULL chars, this would trigger a false positive: if you expect that
     // to be the case, take extra precautions.
-    bool move_to_key(const char * key);
+    inline bool move_to_key(const char *key);
+    // when at {, go one level deep, looking for a given key
+    // if successful, we are left pointing at the value,
+    // if not, we are still pointing at the object ({)
+    // (in case of repeated keys, this only finds the first one).
+    // The string we search for can contain NULL values.
+    inline bool move_to_key(const char *key, uint32_t length);
 
+    // when at a key location within an object, this moves to the accompanying
+    // value (located next to it). this is equivalent but much faster than
+    // calling "next()".
+    inline void move_to_value();
+
+    // when at [, go one level deep, and advance to the given index.
+    // if successful, we are left pointing at the value,
+    // if not, we are still pointing at the array ([)
+    inline bool move_to_index(uint32_t index);
+
+    // Moves the iterator to the value correspoding to the json pointer.
+    // Always search from the root of the document.
+    // if successful, we are left pointing at the value,
+    // if not, we are still pointing the same value we were pointing before the
+    // call. The json pointer follows the rfc6901 standard's syntax:
+    // https://tools.ietf.org/html/rfc6901 However, the standard says "If a
+    // referenced member name is not unique in an object, the member that is
+    // referenced is undefined, and evaluation fails". Here we just return the
+    // first corresponding value. The length parameter is the length of the
+    // jsonpointer string ('pointer').
+    bool move_to(const char *pointer, uint32_t length);
+
+    // Moves the iterator to the value correspoding to the json pointer.
+    // Always search from the root of the document.
+    // if successful, we are left pointing at the value,
+    // if not, we are still pointing the same value we were pointing before the
+    // call. The json pointer implementation follows the rfc6901 standard's
+    // syntax: https://tools.ietf.org/html/rfc6901 However, the standard says
+    // "If a referenced member name is not unique in an object, the member that
+    // is referenced is undefined, and evaluation fails". Here we just return
+    // the first corresponding value.
+    inline bool move_to(const std::string &pointer) {
+      return move_to(pointer.c_str(), pointer.length());
+    }
+
+  private:
+    // Almost the same as move_to(), except it searchs from the current
+    // position. The pointer's syntax is identical, though that case is not
+    // handled by the rfc6901 standard. The '/' is still required at the
+    // beginning. However, contrary to move_to(), the URI Fragment Identifier
+    // Representation is not supported here. Also, in case of failure, we are
+    // left pointing at the closest value it could reach. For these reasons it
+    // is private. It exists because it is used by move_to().
+    bool relative_move_to(const char *pointer, uint32_t length);
+
+  public:
     // throughout return true if we can do the navigation, false
     // otherwise
 
     // Withing a given scope (series of nodes at the same depth within either an
     // array or an object), we move forward.
-    // Thus, given [true, null, {"a":1}, [1,2]], we would visit true, null, { and [.
-    // At the object ({) or at the array ([), you can issue a "down" to visit their content.
-    // valid if we're not at the end of a scope (returns true).
-    bool next();
+    // Thus, given [true, null, {"a":1}, [1,2]], we would visit true, null, {
+    // and [. At the object ({) or at the array ([), you can issue a "down" to
+    // visit their content. valid if we're not at the end of a scope (returns
+    // true).
+    inline bool next();
 
     // Withing a given scope (series of nodes at the same depth within either an
     // array or an object), we move backward.
-    // Thus, given [true, null, {"a":1}, [1,2]], we would visit ], }, null, true when starting at the end
-    // of the scope.
-    // At the object ({) or at the array ([), you can issue a "down" to visit their content.
-    bool prev();
+    // Thus, given [true, null, {"a":1}, [1,2]], we would visit ], }, null, true
+    // when starting at the end of the scope. At the object ({) or at the array
+    // ([), you can issue a "down" to visit their content.
+    inline bool prev();
 
     // Moves back to either the containing array or object (type { or [) from
     // within a contained scope.
     // Valid unless we are at the first level of the document
-    bool up();
+    inline bool up();
 
-
-    // Valid if we're at a [ or { and it starts a non-empty scope; moves us to start of
-    // that deeper scope if it not empty.
-    // Thus, given [true, null, {"a":1}, [1,2]], if we are at the { node, we would move to the
-    // "a" node.
-    bool down();
+    // Valid if we're at a [ or { and it starts a non-empty scope; moves us to
+    // start of that deeper scope if it not empty. Thus, given [true, null,
+    // {"a":1}, [1,2]], if we are at the { node, we would move to the "a" node.
+    inline bool down();
 
     // move us to the start of our current scope,
     // a scope is a series of nodes at the same level
-    void to_start_scope();
+    inline void to_start_scope();
+
+    inline void rewind() {
+      while (up())
+        ;
+    }
 
     // void to_end_scope();              // move us to
     // the start of our current scope; always succeeds
 
     // print the thing we're currently pointing at
     bool print(std::ostream &os, bool escape_strings = true) const;
-    typedef struct {size_t start_of_scope; uint8_t scope_type;} scopeindex_t;
+    typedef struct {
+      size_t start_of_scope;
+      uint8_t scope_type;
+    } scopeindex_t;
 
-private:
-
-    iterator& operator=(const iterator& other) = delete ;
+  private:
+    Iterator &operator=(const Iterator &other) = delete;
 
     ParsedJson &pj;
     size_t depth;
-    size_t location;     // our current location on a tape
+    size_t location; // our current location on a tape
     size_t tape_length;
     uint8_t current_type;
     uint64_t current_val;
-    scopeindex_t *depthindex;
+    scopeindex_t *depth_index;
   };
 
-  size_t bytecapacity{0};  // indicates how many bits are meant to be supported 
+  size_t byte_capacity{0}; // indicates how many bits are meant to be supported
 
-  size_t depthcapacity{0}; // how deep we can go
-  size_t tapecapacity{0};
-  size_t stringcapacity{0};
+  size_t depth_capacity{0}; // how deep we can go
+  size_t tape_capacity{0};
+  size_t string_capacity{0};
   uint32_t current_loc{0};
   uint32_t n_structural_indexes{0};
 
@@ -225,23 +332,23 @@ private:
   uint32_t *containing_scope_offset;
 #ifdef SIMDJSON_USE_COMPUTED_GOTO
   void **ret_address;
-#else 
+#else
   char *ret_address;
 #endif
 
-  uint8_t *string_buf; // should be at least bytecapacity
+  uint8_t *string_buf; // should be at least byte_capacity
   uint8_t *current_string_buf_loc;
-  bool isvalid{false};
+  bool valid{false};
+  int error_code{simdjson::UNITIALIZED};
 
-private :
-
- // we don't want the default constructor to be called
- ParsedJson(const ParsedJson & p) = delete; // we don't want the default constructor to be called
- // we don't want the assignment to be called
- ParsedJson & operator=(const ParsedJson&o) = delete;
+private:
+  // we don't want the default constructor to be called
+  ParsedJson(const ParsedJson &p) =
+      delete; // we don't want the default constructor to be called
+  // we don't want the assignment to be called
+  ParsedJson &operator=(const ParsedJson &o) = delete;
 };
 
-
 // dump bits low to high
 inline void dumpbits_always(uint64_t v, const std::string &msg) {
   for (uint32_t i = 0; i < 64; i++) {
@@ -257,5 +364,181 @@ inline void dumpbits32_always(uint32_t v, const std::string &msg) {
   std::cout << " " << msg.c_str() << "\n";
 }
 
+WARN_UNUSED
+bool ParsedJson::Iterator::is_ok() const { return location < tape_length; }
 
+// useful for debuging purposes
+size_t ParsedJson::Iterator::get_tape_location() const { return location; }
+
+// useful for debuging purposes
+size_t ParsedJson::Iterator::get_tape_length() const { return tape_length; }
+
+// returns the current depth (start at 1 with 0 reserved for the fictitious root
+// node)
+size_t ParsedJson::Iterator::get_depth() const { return depth; }
+
+// A scope is a series of nodes at the same depth, typically it is either an
+// object ({) or an array ([). The root node has type 'r'.
+uint8_t ParsedJson::Iterator::get_scope_type() const {
+  return depth_index[depth].scope_type;
+}
+
+bool ParsedJson::Iterator::move_forward() {
+  if (location + 1 >= tape_length) {
+    return false; // we are at the end!
+  }
+
+  if ((current_type == '[') || (current_type == '{')) {
+    // We are entering a new scope
+    depth++;
+    depth_index[depth].start_of_scope = location;
+    depth_index[depth].scope_type = current_type;
+  } else if ((current_type == ']') || (current_type == '}')) {
+    // Leaving a scope.
+    depth--;
+  } else if ((current_type == 'd') || (current_type == 'l')) {
+    // d and l types use 2 locations on the tape, not just one.
+    location += 1;
+  }
+
+  location += 1;
+  current_val = pj.tape[location];
+  current_type = (current_val >> 56);
+  return true;
+}
+
+void ParsedJson::Iterator::move_to_value() {
+  // assume that we are on a key, so move by 1.
+  location += 1;
+  current_val = pj.tape[location];
+  current_type = (current_val >> 56);
+}
+
+bool ParsedJson::Iterator::move_to_key(const char *key) {
+  if (down()) {
+    do {
+      assert(is_string());
+      bool right_key =
+          (strcmp(get_string(), key) == 0); // null chars would fool this
+      move_to_value();
+      if (right_key) {
+        return true;
+      }
+    } while (next());
+    assert(up()); // not found
+  }
+  return false;
+}
+
+bool ParsedJson::Iterator::move_to_key(const char *key, uint32_t length) {
+  if (down()) {
+    do {
+      assert(is_string());
+      bool right_key = ((get_string_length() == length) &&
+                        (memcmp(get_string(), key, length) == 0));
+      move_to_value();
+      if (right_key) {
+        return true;
+      }
+    } while (next());
+    assert(up()); // not found
+  }
+  return false;
+}
+
+bool ParsedJson::Iterator::move_to_index(uint32_t index) {
+  assert(is_array());
+  if (down()) {
+    uint32_t i = 0;
+    for (; i < index; i++) {
+      if (!next()) {
+        break;
+      }
+    }
+    if (i == index) {
+      return true;
+    }
+    assert(up());
+  }
+  return false;
+}
+
+bool ParsedJson::Iterator::prev() {
+  if (location - 1 < depth_index[depth].start_of_scope) {
+    return false;
+  }
+  location -= 1;
+  current_val = pj.tape[location];
+  current_type = (current_val >> 56);
+  if ((current_type == ']') || (current_type == '}')) {
+    // we need to jump
+    size_t new_location = (current_val & JSON_VALUE_MASK);
+    if (new_location < depth_index[depth].start_of_scope) {
+      return false; // shoud never happen
+    }
+    location = new_location;
+    current_val = pj.tape[location];
+    current_type = (current_val >> 56);
+  }
+  return true;
+}
+
+bool ParsedJson::Iterator::up() {
+  if (depth == 1) {
+    return false; // don't allow moving back to root
+  }
+  to_start_scope();
+  // next we just move to the previous value
+  depth--;
+  location -= 1;
+  current_val = pj.tape[location];
+  current_type = (current_val >> 56);
+  return true;
+}
+
+bool ParsedJson::Iterator::down() {
+  if (location + 1 >= tape_length) {
+    return false;
+  }
+  if ((current_type == '[') || (current_type == '{')) {
+    size_t npos = (current_val & JSON_VALUE_MASK);
+    if (npos == location + 2) {
+      return false; // we have an empty scope
+    }
+    depth++;
+    location = location + 1;
+    depth_index[depth].start_of_scope = location;
+    depth_index[depth].scope_type = current_type;
+    current_val = pj.tape[location];
+    current_type = (current_val >> 56);
+    return true;
+  }
+  return false;
+}
+
+void ParsedJson::Iterator::to_start_scope() {
+  location = depth_index[depth].start_of_scope;
+  current_val = pj.tape[location];
+  current_type = (current_val >> 56);
+}
+
+bool ParsedJson::Iterator::next() {
+  size_t npos;
+  if ((current_type == '[') || (current_type == '{')) {
+    // we need to jump
+    npos = (current_val & JSON_VALUE_MASK);
+  } else {
+    npos = location + ((current_type == 'd' || current_type == 'l') ? 2 : 1);
+  }
+  uint64_t next_val = pj.tape[npos];
+  uint8_t next_type = (next_val >> 56);
+  if ((next_type == ']') || (next_type == '}')) {
+    return false; // we reached the end of the scope
+  }
+  location = npos;
+  current_val = next_val;
+  current_type = next_type;
+  return true;
+}
+} // namespace simdjson
 #endif

include/simdjson/portability.h

@@ -1,127 +1,172 @@
 #ifndef SIMDJSON_PORTABILITY_H
 #define SIMDJSON_PORTABILITY_H
 
-#if defined(_MSC_VER)
+#if defined(__x86_64__) || defined(_M_AMD64)
+#define IS_X86_64 1
+#endif
+#if defined(__aarch64__) || defined(_M_ARM64)
+#define IS_ARM64 1
+#endif
+
+// this is almost standard?
+#define STRINGIFY(a) #a
+
+// we are going to use runtime dispatch
+#ifdef IS_X86_64
+#ifdef __clang__
+// clang does not have GCC push pop
+// warning: clang attribute push can't be used within a namespace in clang up
+// til 8.0 so TARGET_REGION and UNTARGET_REGION must be *outside* of a
+// namespace.
+#define TARGET_REGION(T)                                                       \
+  _Pragma(STRINGIFY(                                                           \
+      clang attribute push(__attribute__((target(T))), apply_to = function)))
+#define UNTARGET_REGION _Pragma("clang attribute pop")
+#elif defined(__GNUC__)
+// GCC is easier
+#define TARGET_REGION(T)                                                       \
+  _Pragma("GCC push_options") _Pragma(STRINGIFY(GCC target(T)))
+#define UNTARGET_REGION _Pragma("GCC pop_options")
+#else
+#define TARGET_REGION(T)
+#define UNTARGET_REGION
+#endif // clang then gcc
+
+// under GCC and CLANG, we use these two macros
+#define TARGET_HASWELL TARGET_REGION("avx2,bmi,pclmul")
+#define TARGET_WESTMERE TARGET_REGION("sse4.2,pclmul")
+
+#endif // x86
+
+#ifdef _MSC_VER
 #include <intrin.h>
 #else
+#if IS_X86_64
 #include <x86intrin.h>
+#elif IS_ARM64
+#include <arm_neon.h>
+#endif
 #endif
 
 #ifdef _MSC_VER
 /* Microsoft C/C++-compatible compiler */
-#include <intrin.h>
+#include <cstdint>
 #include <iso646.h>
-#include <cstdint>
 
-static inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) {
-	return _addcarry_u64(0, value1, value2, reinterpret_cast<unsigned __int64 *>(result));
+namespace simdjson {
+static inline bool add_overflow(uint64_t value1, uint64_t value2,
+                                uint64_t *result) {
+  return _addcarry_u64(0, value1, value2,
+                       reinterpret_cast<unsigned __int64 *>(result));
 }
 
-#  pragma intrinsic(_umul128)
-static inline bool mul_overflow(uint64_t value1, uint64_t value2, uint64_t *result) {
-	uint64_t high;
-	*result = _umul128(value1, value2, &high);
-	return high;
+#pragma intrinsic(_umul128)
+static inline bool mul_overflow(uint64_t value1, uint64_t value2,
+                                uint64_t *result) {
+  uint64_t high;
+  *result = _umul128(value1, value2, &high);
+  return high;
 }
 
-static inline int trailingzeroes(uint64_t input_num) {
-    return _tzcnt_u64(input_num);
+static inline int trailing_zeroes(uint64_t input_num) {
+  return static_cast<int>(_tzcnt_u64(input_num));
 }
 
-static inline int leadingzeroes(uint64_t  input_num) {
-    return _lzcnt_u64(input_num);
+static inline int leading_zeroes(uint64_t input_num) {
+  return static_cast<int>(_lzcnt_u64(input_num));
 }
 
 static inline int hamming(uint64_t input_num) {
-#ifdef _WIN64  // highly recommended!!!
-	return (int)__popcnt64(input_num);
-#else  // if we must support 32-bit Windows
-	return (int)(__popcnt((uint32_t)input_num) +
-		__popcnt((uint32_t)(input_num >> 32)));
+#ifdef _WIN64 // highly recommended!!!
+  return (int)__popcnt64(input_num);
+#else // if we must support 32-bit Windows
+  return (int)(__popcnt((uint32_t)input_num) +
+               __popcnt((uint32_t)(input_num >> 32)));
 #endif
 }
-
+} // namespace simdjson
 #else
 #include <cstdint>
-#include <x86intrin.h>
+#include <cstdlib>
 
-static inline bool add_overflow(uint64_t  value1, uint64_t  value2, uint64_t *result) {
-	return __builtin_uaddll_overflow(value1, value2, (unsigned long long*)result);
+namespace simdjson {
+static inline bool add_overflow(uint64_t value1, uint64_t value2,
+                                uint64_t *result) {
+  return __builtin_uaddll_overflow(value1, value2,
+                                   (unsigned long long *)result);
 }
-static inline bool mul_overflow(uint64_t  value1, uint64_t  value2, uint64_t *result) {
-	return __builtin_umulll_overflow(value1, value2, (unsigned long long *)result);
+static inline bool mul_overflow(uint64_t value1, uint64_t value2,
+                                uint64_t *result) {
+  return __builtin_umulll_overflow(value1, value2,
+                                   (unsigned long long *)result);
 }
 
 /* result might be undefined when input_num is zero */
-static inline int trailingzeroes(uint64_t input_num) {
-#ifdef __BMI__
-	return _tzcnt_u64(input_num);
+static inline int trailing_zeroes(uint64_t input_num) {
+#ifdef __BMI__ // tzcnt is BMI1
+  return _tzcnt_u64(input_num);
 #else
-#warning "BMI is missing?"
-	return __builtin_ctzll(input_num);
+  return __builtin_ctzll(input_num);
 #endif
 }
 
 /* result might be undefined when input_num is zero */
-static inline int leadingzeroes(uint64_t  input_num) {
-	return _lzcnt_u64(input_num);
+static inline int leading_zeroes(uint64_t input_num) {
+#ifdef __BMI2__
+  return _lzcnt_u64(input_num);
+#else
+  return __builtin_clzll(input_num);
+#endif
 }
 
 /* result might be undefined when input_num is zero */
 static inline int hamming(uint64_t input_num) {
-	return _popcnt64(input_num);
+#ifdef __POPCOUNT__
+  return _popcnt64(input_num);
+#else
+  return __builtin_popcountll(input_num);
+#endif
 }
-
+} // namespace simdjson
 #endif // _MSC_VER
 
-
-
+namespace simdjson {
 // portable version of  posix_memalign
 static inline void *aligned_malloc(size_t alignment, size_t size) {
-	void *p;
+  void *p;
 #ifdef _MSC_VER
-	p = _aligned_malloc(size, alignment);
+  p = _aligned_malloc(size, alignment);
 #elif defined(__MINGW32__) || defined(__MINGW64__)
-	p = __mingw_aligned_malloc(size, alignment);
+  p = __mingw_aligned_malloc(size, alignment);
 #else
-	// somehow, if this is used before including "x86intrin.h", it creates an
-	// implicit defined warning.
-	if (posix_memalign(&p, alignment, size) != 0) { return nullptr; }
+  // somehow, if this is used before including "x86intrin.h", it creates an
+  // implicit defined warning.
+  if (posix_memalign(&p, alignment, size) != 0) {
+    return nullptr;
+  }
 #endif
-	return p;
+  return p;
 }
 
-
-#ifndef __clang__
-#ifndef _MSC_VER
-static __m256i inline _mm256_loadu2_m128i(__m128i const *__addr_hi,
-                                          __m128i const *__addr_lo) {
-  __m256i __v256 = _mm256_castsi128_si256(_mm_loadu_si128(__addr_lo));
-  return _mm256_insertf128_si256(__v256, _mm_loadu_si128(__addr_hi), 1);
+static inline char *aligned_malloc_char(size_t alignment, size_t size) {
+  return (char *)aligned_malloc(alignment, size);
 }
 
-static inline void _mm256_storeu2_m128i(__m128i *__addr_hi, __m128i *__addr_lo,
-                                        __m256i __a) {
-  __m128i __v128;
-
-  __v128 = _mm256_castsi256_si128(__a);
-  _mm_storeu_si128(__addr_lo, __v128);
-  __v128 = _mm256_extractf128_si256(__a, 1);
-  _mm_storeu_si128(__addr_hi, __v128);
-}
-#endif
-#endif
-
-
-static inline void aligned_free(void *memblock) {
-    if(memblock == nullptr) { return; }
+static inline void aligned_free(void *mem_block) {
+  if (mem_block == nullptr) {
+    return;
+  }
 #ifdef _MSC_VER
-    _aligned_free(memblock);
+  _aligned_free(mem_block);
 #elif defined(__MINGW32__) || defined(__MINGW64__)
-    __mingw_aligned_free(memblock);
+  __mingw_aligned_free(mem_block);
 #else
-    free(memblock);
+  free(mem_block);
 #endif
 }
 
+static inline void aligned_free_char(char *mem_block) {
+  aligned_free((void *)mem_block);
+}
+} // namespace simdjson
 #endif // SIMDJSON_PORTABILITY_H

include/simdjson/simdjson.h

@@ -1,16 +1,44 @@
-#ifndef  SIMDJSON_ERR_H
-# define SIMDJSON_ERR_H
+#ifndef SIMDJSON_ERR_H
+#define SIMDJSON_ERR_H
 
 #include <string>
 
-struct simdjson {
-  enum errorValues {
-    SUCCESS = 0,
-    CAPACITY, // This ParsedJson can't support a document that big
-    MEMALLOC, // Error allocating memory, most likely out of memory
-    TAPE_ERROR, // Something went wrong while writing to the tape
-  };
-  static const std::string& errorMsg(const int);
+namespace simdjson {
+// Represents the minimal architecture that would support an implementation
+enum class Architecture {
+  WESTMERE,
+  HASWELL,
+  ARM64,
+  NONE,
+// TODO remove 'native' in favor of runtime dispatch?
+// the 'native' enum class value should point at a good default on the current
+// machine
+#ifdef IS_X86_64
+  NATIVE = WESTMERE
+#elif defined(IS_ARM64)
+  NATIVE = ARM64
+#endif
 };
 
-#endif
+enum ErrorValues {
+  SUCCESS = 0,
+  CAPACITY,    // This ParsedJson can't support a document that big
+  MEMALLOC,    // Error allocating memory, most likely out of memory
+  TAPE_ERROR,  // Something went wrong while writing to the tape (stage 2), this
+               // is a generic error
+  DEPTH_ERROR, // Your document exceeds the user-specified depth limitation
+  STRING_ERROR,    // Problem while parsing a string
+  T_ATOM_ERROR,    // Problem while parsing an atom starting with the letter 't'
+  F_ATOM_ERROR,    // Problem while parsing an atom starting with the letter 'f'
+  N_ATOM_ERROR,    // Problem while parsing an atom starting with the letter 'n'
+  NUMBER_ERROR,    // Problem while parsing a number
+  UTF8_ERROR,      // the input is not valid UTF-8
+  UNITIALIZED,     // unknown error, or uninitialized document
+  EMPTY,           // no structural document found
+  UNESCAPED_CHARS, // found unescaped characters in a string.
+  UNCLOSED_STRING, // missing quote at the end
+  UNEXPECTED_ERROR // indicative of a bug in simdjson
+};
+const std::string &error_message(const int);
+} // namespace simdjson
+#endif

include/simdjson/simdjson_version.h

@@ -1,10 +1,13 @@
-// /include/simdjson/simdjson_version.h automatically generated by release.py, do not change by hand 
-#ifndef SIMDJSON_INCLUDE_SIMDJSON_VERSION 
-#define SIMDJSON_INCLUDE_SIMDJSON_VERSION 
-#define SIMDJSON_VERSION 0.1.0 
-enum { 
-    SIMDJSON_VERSION_MAJOR = 0,  
-    SIMDJSON_VERSION_MINOR = 1,  
-    SIMDJSON_VERSION_REVISION = 0  
-}; 
-#endif // SIMDJSON_INCLUDE_SIMDJSON_VERSION 
+// /include/simdjson/simdjson_version.h automatically generated by release.py,
+// do not change by hand
+#ifndef SIMDJSON_INCLUDE_SIMDJSON_VERSION
+#define SIMDJSON_INCLUDE_SIMDJSON_VERSION
+#define SIMDJSON_VERSION 0.2.1
+namespace simdjson {
+enum {
+  SIMDJSON_VERSION_MAJOR = 0,
+  SIMDJSON_VERSION_MINOR = 2,
+  SIMDJSON_VERSION_REVISION = 1
+};
+}
+#endif // SIMDJSON_INCLUDE_SIMDJSON_VERSION

include/simdjson/simdprune_tables.h

@@ -1,10 +1,10 @@
 #ifndef SIMDJSON_SIMDPRUNE_TABLES_H
 #define SIMDJSON_SIMDPRUNE_TABLES_H
+#include <cstdint>
 
 #include "simdjson/portability.h"
 
-#ifdef __AVX__
-
+namespace simdjson {
 static const unsigned char mask128_epi8[] = {
     0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe,
     0xf, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe,
@@ -34984,13 +34984,9 @@ static const unsigned char mask128_epi32[] = {
     0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
     0xff, 0xff, 0xff, 0xff,
 };
+} // namespace simdjson
 
-#endif //__SSE3__
-
-#ifdef __AVX2__
-
-#include <cstdint>
-
+namespace simdjson {
 static const uint32_t mask256_epi32[] = {
     0, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 7, 0, 2, 3, 4, 5, 6, 7, 7, 2,
     3, 4, 5, 6, 7, 7, 7, 0, 1, 3, 4, 5, 6, 7, 7, 1, 3, 4, 5, 6, 7, 7, 7, 0, 3,
@@ -35074,6 +35070,5 @@ static const uint32_t mask256_epi32[] = {
     3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2,
     0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 1, 1, 1, 1, 1, 1, 1, 1,
     1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-#endif //__AVX2__
-
+}
 #endif

include/simdjson/simdutf8check_arm64.h

@@ -0,0 +1,180 @@
+// From https://github.com/cyb70289/utf8/blob/master/lemire-neon.c
+// Adapted from https://github.com/lemire/fastvalidate-utf-8
+
+#ifndef SIMDJSON_SIMDUTF8CHECK_ARM64_H
+#define SIMDJSON_SIMDUTF8CHECK_ARM64_H
+
+#if defined(_ARM_NEON) || defined(__aarch64__) ||                              \
+    (defined(_MSC_VER) && defined(_M_ARM64))
+
+#include <arm_neon.h>
+#include <cinttypes>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+
+/*
+ * legal utf-8 byte sequence
+ * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94
+ *
+ *  Code Points        1st       2s       3s       4s
+ * U+0000..U+007F     00..7F
+ * U+0080..U+07FF     C2..DF   80..BF
+ * U+0800..U+0FFF     E0       A0..BF   80..BF
+ * U+1000..U+CFFF     E1..EC   80..BF   80..BF
+ * U+D000..U+D7FF     ED       80..9F   80..BF
+ * U+E000..U+FFFF     EE..EF   80..BF   80..BF
+ * U+10000..U+3FFFF   F0       90..BF   80..BF   80..BF
+ * U+40000..U+FFFFF   F1..F3   80..BF   80..BF   80..BF
+ * U+100000..U+10FFFF F4       80..8F   80..BF   80..BF
+ *
+ */
+namespace simdjson {
+
+// all byte values must be no larger than 0xF4
+static inline void check_smaller_than_0xF4(int8x16_t current_bytes,
+                                           int8x16_t *has_error) {
+  // unsigned, saturates to 0 below max
+  *has_error = vorrq_s8(
+      *has_error, vreinterpretq_s8_u8(vqsubq_u8(
+                      vreinterpretq_u8_s8(current_bytes), vdupq_n_u8(0xF4))));
+}
+
+static const int8_t _nibbles[] = {
+    1, 1, 1, 1, 1, 1, 1, 1, // 0xxx (ASCII)
+    0, 0, 0, 0,             // 10xx (continuation)
+    2, 2,                   // 110x
+    3,                      // 1110
+    4,                      // 1111, next should be 0 (not checked here)
+};
+
+static inline int8x16_t continuation_lengths(int8x16_t high_nibbles) {
+  return vqtbl1q_s8(vld1q_s8(_nibbles), vreinterpretq_u8_s8(high_nibbles));
+}
+
+static inline int8x16_t carry_continuations(int8x16_t initial_lengths,
+                                            int8x16_t previous_carries) {
+
+  int8x16_t right1 = vreinterpretq_s8_u8(vqsubq_u8(
+      vreinterpretq_u8_s8(vextq_s8(previous_carries, initial_lengths, 16 - 1)),
+      vdupq_n_u8(1)));
+  int8x16_t sum = vaddq_s8(initial_lengths, right1);
+
+  int8x16_t right2 = vreinterpretq_s8_u8(
+      vqsubq_u8(vreinterpretq_u8_s8(vextq_s8(previous_carries, sum, 16 - 2)),
+                vdupq_n_u8(2)));
+  return vaddq_s8(sum, right2);
+}
+
+static inline void check_continuations(int8x16_t initial_lengths,
+                                       int8x16_t carries,
+                                       int8x16_t *has_error) {
+
+  // overlap || underlap
+  // carry > length && length > 0 || !(carry > length) && !(length > 0)
+  // (carries > length) == (lengths > 0)
+  uint8x16_t overunder = vceqq_u8(vcgtq_s8(carries, initial_lengths),
+                                  vcgtq_s8(initial_lengths, vdupq_n_s8(0)));
+
+  *has_error = vorrq_s8(*has_error, vreinterpretq_s8_u8(overunder));
+}
+
+// when 0xED is found, next byte must be no larger than 0x9F
+// when 0xF4 is found, next byte must be no larger than 0x8F
+// next byte must be continuation, ie sign bit is set, so signed < is ok
+static inline void check_first_continuation_max(int8x16_t current_bytes,
+                                                int8x16_t off1_current_bytes,
+                                                int8x16_t *has_error) {
+  uint8x16_t maskED = vceqq_s8(off1_current_bytes, vdupq_n_s8(0xED));
+  uint8x16_t maskF4 = vceqq_s8(off1_current_bytes, vdupq_n_s8(0xF4));
+
+  uint8x16_t badfollowED =
+      vandq_u8(vcgtq_s8(current_bytes, vdupq_n_s8(0x9F)), maskED);
+  uint8x16_t badfollowF4 =
+      vandq_u8(vcgtq_s8(current_bytes, vdupq_n_s8(0x8F)), maskF4);
+
+  *has_error = vorrq_s8(
+      *has_error, vreinterpretq_s8_u8(vorrq_u8(badfollowED, badfollowF4)));
+}
+
+static const int8_t _initial_mins[] = {
+    -128,         -128, -128, -128, -128, -128,
+    -128,         -128, -128, -128, -128, -128, // 10xx => false
+    (int8_t)0xC2, -128,                         // 110x
+    (int8_t)0xE1,                               // 1110
+    (int8_t)0xF1,
+};
+
+static const int8_t _second_mins[] = {
+    -128,         -128, -128, -128, -128, -128,
+    -128,         -128, -128, -128, -128, -128, // 10xx => false
+    127,          127,                          // 110x => true
+    (int8_t)0xA0,                               // 1110
+    (int8_t)0x90,
+};
+
+// map off1_hibits => error condition
+// hibits     off1    cur
+// C       => < C2 && true
+// E       => < E1 && < A0
+// F       => < F1 && < 90
+// else      false && false
+static inline void check_overlong(int8x16_t current_bytes,
+                                  int8x16_t off1_current_bytes,
+                                  int8x16_t hibits, int8x16_t previous_hibits,
+                                  int8x16_t *has_error) {
+  int8x16_t off1_hibits = vextq_s8(previous_hibits, hibits, 16 - 1);
+  int8x16_t initial_mins =
+      vqtbl1q_s8(vld1q_s8(_initial_mins), vreinterpretq_u8_s8(off1_hibits));
+
+  uint8x16_t initial_under = vcgtq_s8(initial_mins, off1_current_bytes);
+
+  int8x16_t second_mins =
+      vqtbl1q_s8(vld1q_s8(_second_mins), vreinterpretq_u8_s8(off1_hibits));
+  uint8x16_t second_under = vcgtq_s8(second_mins, current_bytes);
+  *has_error = vorrq_s8(
+      *has_error, vreinterpretq_s8_u8(vandq_u8(initial_under, second_under)));
+}
+
+struct processed_utf_bytes {
+  int8x16_t raw_bytes;
+  int8x16_t high_nibbles;
+  int8x16_t carried_continuations;
+};
+
+static inline void count_nibbles(int8x16_t bytes,
+                                 struct processed_utf_bytes *answer) {
+  answer->raw_bytes = bytes;
+  answer->high_nibbles =
+      vreinterpretq_s8_u8(vshrq_n_u8(vreinterpretq_u8_s8(bytes), 4));
+}
+
+// check whether the current bytes are valid UTF-8
+// at the end of the function, previous gets updated
+static inline struct processed_utf_bytes
+check_utf8_bytes(int8x16_t current_bytes, struct processed_utf_bytes *previous,
+                 int8x16_t *has_error) {
+  struct processed_utf_bytes pb;
+  count_nibbles(current_bytes, &pb);
+
+  check_smaller_than_0xF4(current_bytes, has_error);
+
+  int8x16_t initial_lengths = continuation_lengths(pb.high_nibbles);
+
+  pb.carried_continuations =
+      carry_continuations(initial_lengths, previous->carried_continuations);
+
+  check_continuations(initial_lengths, pb.carried_continuations, has_error);
+
+  int8x16_t off1_current_bytes =
+      vextq_s8(previous->raw_bytes, pb.raw_bytes, 16 - 1);
+  check_first_continuation_max(current_bytes, off1_current_bytes, has_error);
+
+  check_overlong(current_bytes, off1_current_bytes, pb.high_nibbles,
+                 previous->high_nibbles, has_error);
+  return pb;
+}
+} // namespace simdjson
+#endif
+#endif

include/simdjson/simdutf8check_haswell.h

@@ -1,12 +1,12 @@
+#ifndef SIMDJSON_SIMDUTF8CHECK_HASWELL_H
+#define SIMDJSON_SIMDUTF8CHECK_HASWELL_H
 
-#ifndef SIMDJSON_SIMDUTF8CHECK_H
-#define SIMDJSON_SIMDUTF8CHECK_H
-
-
+#include "simdjson/portability.h"
 #include <stddef.h>
 #include <stdint.h>
 #include <string.h>
-#include "simdjson/portability.h"
+
+#ifdef IS_X86_64
 /*
  * legal utf-8 byte sequence
  * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94
@@ -26,10 +26,8 @@
 
 // all byte values must be no larger than 0xF4
 
-
-#ifdef __AVX2__
-
-/*****************************/
+TARGET_HASWELL
+namespace simdjson {
 static inline __m256i push_last_byte_of_a_to_b(__m256i a, __m256i b) {
   return _mm256_alignr_epi8(b, _mm256_permute2x128_si256(a, b, 0x21), 15);
 }
@@ -39,14 +37,14 @@ static inline __m256i push_last_2bytes_of_a_to_b(__m256i a, __m256i b) {
 }
 
 // all byte values must be no larger than 0xF4
-static inline void avxcheckSmallerThan0xF4(__m256i current_bytes,
-                                           __m256i *has_error) {
+static inline void avx_check_smaller_than_0xF4(__m256i current_bytes,
+                                               __m256i *has_error) {
   // unsigned, saturates to 0 below max
   *has_error = _mm256_or_si256(
-      *has_error, _mm256_subs_epu8(current_bytes, _mm256_set1_epi8(0xF4)));
+      *has_error, _mm256_subs_epu8(current_bytes, _mm256_set1_epi8(0xF4u)));
 }
 
-static inline __m256i avxcontinuationLengths(__m256i high_nibbles) {
+static inline __m256i avx_continuation_lengths(__m256i high_nibbles) {
   return _mm256_shuffle_epi8(
       _mm256_setr_epi8(1, 1, 1, 1, 1, 1, 1, 1, // 0xxx (ASCII)
                        0, 0, 0, 0,             // 10xx (continuation)
@@ -62,8 +60,8 @@ static inline __m256i avxcontinuationLengths(__m256i high_nibbles) {
       high_nibbles);
 }
 
-static inline __m256i avxcarryContinuations(__m256i initial_lengths,
-                                            __m256i previous_carries) {
+static inline __m256i avx_carry_continuations(__m256i initial_lengths,
+                                              __m256i previous_carries) {
 
   __m256i right1 = _mm256_subs_epu8(
       push_last_byte_of_a_to_b(previous_carries, initial_lengths),
@@ -75,8 +73,9 @@ static inline __m256i avxcarryContinuations(__m256i initial_lengths,
   return _mm256_add_epi8(sum, right2);
 }
 
-static inline void avxcheckContinuations(__m256i initial_lengths,
-                                         __m256i carries, __m256i *has_error) {
+static inline void avx_check_continuations(__m256i initial_lengths,
+                                           __m256i carries,
+                                           __m256i *has_error) {
 
   // overlap || underlap
   // carry > length && length > 0 || !(carry > length) && !(length > 0)
@@ -91,18 +90,18 @@ static inline void avxcheckContinuations(__m256i initial_lengths,
 // when 0xED is found, next byte must be no larger than 0x9F
 // when 0xF4 is found, next byte must be no larger than 0x8F
 // next byte must be continuation, ie sign bit is set, so signed < is ok
-static inline void avxcheckFirstContinuationMax(__m256i current_bytes,
-                                                __m256i off1_current_bytes,
-                                                __m256i *has_error) {
+static inline void avx_check_first_continuation_max(__m256i current_bytes,
+                                                    __m256i off1_current_bytes,
+                                                    __m256i *has_error) {
   __m256i maskED =
-      _mm256_cmpeq_epi8(off1_current_bytes, _mm256_set1_epi8(0xED));
+      _mm256_cmpeq_epi8(off1_current_bytes, _mm256_set1_epi8(0xEDu));
   __m256i maskF4 =
-      _mm256_cmpeq_epi8(off1_current_bytes, _mm256_set1_epi8(0xF4));
+      _mm256_cmpeq_epi8(off1_current_bytes, _mm256_set1_epi8(0xF4u));
 
   __m256i badfollowED = _mm256_and_si256(
-      _mm256_cmpgt_epi8(current_bytes, _mm256_set1_epi8(0x9F)), maskED);
+      _mm256_cmpgt_epi8(current_bytes, _mm256_set1_epi8(0x9Fu)), maskED);
   __m256i badfollowF4 = _mm256_and_si256(
-      _mm256_cmpgt_epi8(current_bytes, _mm256_set1_epi8(0x8F)), maskF4);
+      _mm256_cmpgt_epi8(current_bytes, _mm256_set1_epi8(0x8Fu)), maskF4);
 
   *has_error =
       _mm256_or_si256(*has_error, _mm256_or_si256(badfollowED, badfollowF4));
@@ -114,21 +113,22 @@ static inline void avxcheckFirstContinuationMax(__m256i current_bytes,
 // E       => < E1 && < A0
 // F       => < F1 && < 90
 // else      false && false
-static inline void avxcheckOverlong(__m256i current_bytes,
-                                    __m256i off1_current_bytes, __m256i hibits,
-                                    __m256i previous_hibits,
-                                    __m256i *has_error) {
+static inline void avx_check_overlong(__m256i current_bytes,
+                                      __m256i off1_current_bytes,
+                                      __m256i hibits, __m256i previous_hibits,
+                                      __m256i *has_error) {
   __m256i off1_hibits = push_last_byte_of_a_to_b(previous_hibits, hibits);
   __m256i initial_mins = _mm256_shuffle_epi8(
       _mm256_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128, -128,
                        -128, -128, -128, // 10xx => false
-                       0xC2, -128,       // 110x
-                       0xE1,             // 1110
-                       0xF1, -128, -128, -128, -128, -128, -128, -128, -128,
-                       -128, -128, -128, -128, // 10xx => false
-                       0xC2, -128,             // 110x
-                       0xE1,                   // 1110
-                       0xF1),
+                       0xC2u, -128,      // 110x
+                       0xE1u,            // 1110
+                       0xF1u,            // 1111
+                       -128, -128, -128, -128, -128, -128, -128, -128, -128,
+                       -128, -128, -128, // 10xx => false
+                       0xC2u, -128,      // 110x
+                       0xE1u,            // 1110
+                       0xF1u),           // 1111
       off1_hibits);
 
   __m256i initial_under = _mm256_cmpgt_epi8(initial_mins, off1_current_bytes);
@@ -137,12 +137,13 @@ static inline void avxcheckOverlong(__m256i current_bytes,
       _mm256_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128, -128,
                        -128, -128, -128, // 10xx => false
                        127, 127,         // 110x => true
-                       0xA0,             // 1110
-                       0x90, -128, -128, -128, -128, -128, -128, -128, -128,
-                       -128, -128, -128, -128, // 10xx => false
-                       127, 127,               // 110x => true
-                       0xA0,                   // 1110
-                       0x90),
+                       0xA0u,            // 1110
+                       0x90u,            // 1111
+                       -128, -128, -128, -128, -128, -128, -128, -128, -128,
+                       -128, -128, -128, // 10xx => false
+                       127, 127,         // 110x => true
+                       0xA0u,            // 1110
+                       0x90u),           // 1111
       off1_hibits);
   __m256i second_under = _mm256_cmpgt_epi8(second_mins, current_bytes);
   *has_error = _mm256_or_si256(*has_error,
@@ -150,14 +151,14 @@ static inline void avxcheckOverlong(__m256i current_bytes,
 }
 
 struct avx_processed_utf_bytes {
-  __m256i rawbytes;
+  __m256i raw_bytes;
   __m256i high_nibbles;
   __m256i carried_continuations;
 };
 
 static inline void avx_count_nibbles(__m256i bytes,
                                      struct avx_processed_utf_bytes *answer) {
-  answer->rawbytes = bytes;
+  answer->raw_bytes = bytes;
   answer->high_nibbles =
       _mm256_and_si256(_mm256_srli_epi16(bytes, 4), _mm256_set1_epi8(0x0F));
 }
@@ -165,31 +166,33 @@ static inline void avx_count_nibbles(__m256i bytes,
 // check whether the current bytes are valid UTF-8
 // at the end of the function, previous gets updated
 static inline struct avx_processed_utf_bytes
-avxcheckUTF8Bytes(__m256i current_bytes,
-                  struct avx_processed_utf_bytes *previous,
-                  __m256i *has_error) {
-  struct avx_processed_utf_bytes pb{};
+avx_check_utf8_bytes(__m256i current_bytes,
+                     struct avx_processed_utf_bytes *previous,
+                     __m256i *has_error) {
+  struct avx_processed_utf_bytes pb {};
   avx_count_nibbles(current_bytes, &pb);
 
-  avxcheckSmallerThan0xF4(current_bytes, has_error);
+  avx_check_smaller_than_0xF4(current_bytes, has_error);
 
-  __m256i initial_lengths = avxcontinuationLengths(pb.high_nibbles);
+  __m256i initial_lengths = avx_continuation_lengths(pb.high_nibbles);
 
   pb.carried_continuations =
-      avxcarryContinuations(initial_lengths, previous->carried_continuations);
+      avx_carry_continuations(initial_lengths, previous->carried_continuations);
 
-  avxcheckContinuations(initial_lengths, pb.carried_continuations, has_error);
+  avx_check_continuations(initial_lengths, pb.carried_continuations, has_error);
 
   __m256i off1_current_bytes =
-      push_last_byte_of_a_to_b(previous->rawbytes, pb.rawbytes);
-  avxcheckFirstContinuationMax(current_bytes, off1_current_bytes, has_error);
+      push_last_byte_of_a_to_b(previous->raw_bytes, pb.raw_bytes);
+  avx_check_first_continuation_max(current_bytes, off1_current_bytes,
+                                   has_error);
 
-  avxcheckOverlong(current_bytes, off1_current_bytes, pb.high_nibbles,
-                   previous->high_nibbles, has_error);
+  avx_check_overlong(current_bytes, off1_current_bytes, pb.high_nibbles,
+                     previous->high_nibbles, has_error);
   return pb;
 }
+} // namespace simdjson
+UNTARGET_REGION // haswell
+
+#endif // IS_X86_64
 
-#else // __AVX2__
-#warning "We require AVX2 support!"
-#endif // __AVX2__
 #endif

include/simdjson/simdutf8check_westmere.h

@@ -0,0 +1,169 @@
+#ifndef SIMDJSON_SIMDUTF8CHECK_WESTMERE_H
+#define SIMDJSON_SIMDUTF8CHECK_WESTMERE_H
+
+#include "simdjson/portability.h"
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#ifdef IS_X86_64
+
+/*
+ * legal utf-8 byte sequence
+ * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94
+ *
+ *  Code Points        1st       2s       3s       4s
+ * U+0000..U+007F     00..7F
+ * U+0080..U+07FF     C2..DF   80..BF
+ * U+0800..U+0FFF     E0       A0..BF   80..BF
+ * U+1000..U+CFFF     E1..EC   80..BF   80..BF
+ * U+D000..U+D7FF     ED       80..9F   80..BF
+ * U+E000..U+FFFF     EE..EF   80..BF   80..BF
+ * U+10000..U+3FFFF   F0       90..BF   80..BF   80..BF
+ * U+40000..U+FFFFF   F1..F3   80..BF   80..BF   80..BF
+ * U+100000..U+10FFFF F4       80..8F   80..BF   80..BF
+ *
+ */
+
+// all byte values must be no larger than 0xF4
+
+/********** sse code **********/
+TARGET_WESTMERE
+
+namespace simdjson {
+// all byte values must be no larger than 0xF4
+static inline void check_smaller_than_0xF4(__m128i current_bytes,
+                                           __m128i *has_error) {
+  // unsigned, saturates to 0 below max
+  *has_error = _mm_or_si128(*has_error,
+                            _mm_subs_epu8(current_bytes, _mm_set1_epi8(0xF4u)));
+}
+
+static inline __m128i continuation_lengths(__m128i high_nibbles) {
+  return _mm_shuffle_epi8(
+      _mm_setr_epi8(1, 1, 1, 1, 1, 1, 1, 1, // 0xxx (ASCII)
+                    0, 0, 0, 0,             // 10xx (continuation)
+                    2, 2,                   // 110x
+                    3,                      // 1110
+                    4), // 1111, next should be 0 (not checked here)
+      high_nibbles);
+}
+
+static inline __m128i carry_continuations(__m128i initial_lengths,
+                                          __m128i previous_carries) {
+
+  __m128i right1 =
+      _mm_subs_epu8(_mm_alignr_epi8(initial_lengths, previous_carries, 16 - 1),
+                    _mm_set1_epi8(1));
+  __m128i sum = _mm_add_epi8(initial_lengths, right1);
+
+  __m128i right2 = _mm_subs_epu8(_mm_alignr_epi8(sum, previous_carries, 16 - 2),
+                                 _mm_set1_epi8(2));
+  return _mm_add_epi8(sum, right2);
+}
+
+static inline void check_continuations(__m128i initial_lengths, __m128i carries,
+                                       __m128i *has_error) {
+
+  // overlap || underlap
+  // carry > length && length > 0 || !(carry > length) && !(length > 0)
+  // (carries > length) == (lengths > 0)
+  __m128i overunder =
+      _mm_cmpeq_epi8(_mm_cmpgt_epi8(carries, initial_lengths),
+                     _mm_cmpgt_epi8(initial_lengths, _mm_setzero_si128()));
+
+  *has_error = _mm_or_si128(*has_error, overunder);
+}
+
+// when 0xED is found, next byte must be no larger than 0x9F
+// when 0xF4 is found, next byte must be no larger than 0x8F
+// next byte must be continuation, ie sign bit is set, so signed < is ok
+static inline void check_first_continuation_max(__m128i current_bytes,
+                                                __m128i off1_current_bytes,
+                                                __m128i *has_error) {
+  __m128i maskED = _mm_cmpeq_epi8(off1_current_bytes, _mm_set1_epi8(0xEDu));
+  __m128i maskF4 = _mm_cmpeq_epi8(off1_current_bytes, _mm_set1_epi8(0xF4u));
+
+  __m128i badfollowED = _mm_and_si128(
+      _mm_cmpgt_epi8(current_bytes, _mm_set1_epi8(0x9Fu)), maskED);
+  __m128i badfollowF4 = _mm_and_si128(
+      _mm_cmpgt_epi8(current_bytes, _mm_set1_epi8(0x8Fu)), maskF4);
+
+  *has_error = _mm_or_si128(*has_error, _mm_or_si128(badfollowED, badfollowF4));
+}
+
+// map off1_hibits => error condition
+// hibits     off1    cur
+// C       => < C2 && true
+// E       => < E1 && < A0
+// F       => < F1 && < 90
+// else      false && false
+static inline void check_overlong(__m128i current_bytes,
+                                  __m128i off1_current_bytes, __m128i hibits,
+                                  __m128i previous_hibits, __m128i *has_error) {
+  __m128i off1_hibits = _mm_alignr_epi8(hibits, previous_hibits, 16 - 1);
+  __m128i initial_mins = _mm_shuffle_epi8(
+      _mm_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128, -128, -128,
+                    -128, -128,  // 10xx => false
+                    0xC2u, -128, // 110x
+                    0xE1u,       // 1110
+                    0xF1u),
+      off1_hibits);
+
+  __m128i initial_under = _mm_cmpgt_epi8(initial_mins, off1_current_bytes);
+
+  __m128i second_mins = _mm_shuffle_epi8(
+      _mm_setr_epi8(-128, -128, -128, -128, -128, -128, -128, -128, -128, -128,
+                    -128, -128, // 10xx => false
+                    127, 127,   // 110x => true
+                    0xA0u,      // 1110
+                    0x90u),
+      off1_hibits);
+  __m128i second_under = _mm_cmpgt_epi8(second_mins, current_bytes);
+  *has_error =
+      _mm_or_si128(*has_error, _mm_and_si128(initial_under, second_under));
+}
+
+struct processed_utf_bytes {
+  __m128i raw_bytes;
+  __m128i high_nibbles;
+  __m128i carried_continuations;
+};
+
+static inline void count_nibbles(__m128i bytes,
+                                 struct processed_utf_bytes *answer) {
+  answer->raw_bytes = bytes;
+  answer->high_nibbles =
+      _mm_and_si128(_mm_srli_epi16(bytes, 4), _mm_set1_epi8(0x0F));
+}
+
+// check whether the current bytes are valid UTF-8
+// at the end of the function, previous gets updated
+static struct processed_utf_bytes
+check_utf8_bytes(__m128i current_bytes, struct processed_utf_bytes *previous,
+                 __m128i *has_error) {
+  struct processed_utf_bytes pb;
+  count_nibbles(current_bytes, &pb);
+
+  check_smaller_than_0xF4(current_bytes, has_error);
+
+  __m128i initial_lengths = continuation_lengths(pb.high_nibbles);
+
+  pb.carried_continuations =
+      carry_continuations(initial_lengths, previous->carried_continuations);
+
+  check_continuations(initial_lengths, pb.carried_continuations, has_error);
+
+  __m128i off1_current_bytes =
+      _mm_alignr_epi8(pb.raw_bytes, previous->raw_bytes, 16 - 1);
+  check_first_continuation_max(current_bytes, off1_current_bytes, has_error);
+
+  check_overlong(current_bytes, off1_current_bytes, pb.high_nibbles,
+                 previous->high_nibbles, has_error);
+  return pb;
+}
+} // namespace simdjson
+UNTARGET_REGION // westmere
+
+#endif // IS_X86_64
+
+#endif

include/simdjson/stage1_find_marks.h

@@ -2,13 +2,120 @@
 #define SIMDJSON_STAGE1_FIND_MARKS_H
 
 #include "simdjson/common_defs.h"
+#include "simdjson/parsedjson.h"
+#include "simdjson/portability.h"
+#include "simdjson/simdjson.h"
+#include <cassert>
 
-struct ParsedJson;
+namespace simdjson {
 
-WARN_UNUSED
-bool find_structural_bits(const uint8_t *buf, size_t len, ParsedJson &pj);
+template <Architecture> struct simd_input;
 
-WARN_UNUSED
-bool find_structural_bits(const char *buf, size_t len, ParsedJson &pj);
+template <Architecture> uint64_t compute_quote_mask(uint64_t quote_bits);
+
+namespace {
+// for when clmul is unavailable
+[[maybe_unused]] uint64_t portable_compute_quote_mask(uint64_t quote_bits) {
+  uint64_t quote_mask = quote_bits ^ (quote_bits << 1);
+  quote_mask = quote_mask ^ (quote_mask << 2);
+  quote_mask = quote_mask ^ (quote_mask << 4);
+  quote_mask = quote_mask ^ (quote_mask << 8);
+  quote_mask = quote_mask ^ (quote_mask << 16);
+  quote_mask = quote_mask ^ (quote_mask << 32);
+  return quote_mask;
+}
+} // namespace
+
+// Holds the state required to perform check_utf8().
+template <Architecture> struct utf8_checking_state;
+
+template <Architecture T>
+void check_utf8(simd_input<T> in, utf8_checking_state<T> &state);
+
+// Checks if the utf8 validation has found any error.
+template <Architecture T>
+ErrorValues check_utf8_errors(utf8_checking_state<T> &state);
+
+// a straightforward comparison of a mask against input.
+template <Architecture T>
+uint64_t cmp_mask_against_input(simd_input<T> in, uint8_t m);
+
+template <Architecture T> simd_input<T> fill_input(const uint8_t *ptr);
+
+// find all values less than or equal than the content of maxval (using unsigned
+// arithmetic)
+template <Architecture T>
+uint64_t unsigned_lteq_against_input(simd_input<T> in, uint8_t m);
+
+template <Architecture T>
+really_inline uint64_t find_odd_backslash_sequences(
+    simd_input<T> in, uint64_t &prev_iter_ends_odd_backslash);
+
+template <Architecture T>
+really_inline uint64_t find_quote_mask_and_bits(
+    simd_input<T> in, uint64_t odd_ends, uint64_t &prev_iter_inside_quote,
+    uint64_t &quote_bits, uint64_t &error_mask);
+
+// do a 'shufti' to detect structural JSON characters
+// they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c
+// these go into the first 3 buckets of the comparison (1/2/4)
+
+// we are also interested in the four whitespace characters
+// space 0x20, linefeed 0x0a, horizontal tab 0x09 and carriage return 0x0d
+// these go into the next 2 buckets of the comparison (8/16)
+template <Architecture T>
+void find_whitespace_and_structurals(simd_input<T> in, uint64_t &whitespace,
+                                     uint64_t &structurals);
+
+// return a updated structural bit vector with quoted contents cleared out and
+// pseudo-structural characters added to the mask
+// updates prev_iter_ends_pseudo_pred which tells us whether the previous
+// iteration ended on a whitespace or a structural character (which means that
+// the next iteration
+// will have a pseudo-structural character at its start)
+really_inline uint64_t finalize_structurals(
+    uint64_t structurals, uint64_t whitespace, uint64_t quote_mask,
+    uint64_t quote_bits, uint64_t &prev_iter_ends_pseudo_pred) {
+  // mask off anything inside quotes
+  structurals &= ~quote_mask;
+  // add the real quote bits back into our bit_mask as well, so we can
+  // quickly traverse the strings we've spent all this trouble gathering
+  structurals |= quote_bits;
+  // Now, establish "pseudo-structural characters". These are non-whitespace
+  // characters that are (a) outside quotes and (b) have a predecessor that's
+  // either whitespace or a structural character. This means that subsequent
+  // passes will get a chance to encounter the first character of every string
+  // of non-whitespace and, if we're parsing an atom like true/false/null or a
+  // number we can stop at the first whitespace or structural character
+  // following it.
+
+  // a qualified predecessor is something that can happen 1 position before an
+  // pseudo-structural character
+  uint64_t pseudo_pred = structurals | whitespace;
+
+  uint64_t shifted_pseudo_pred =
+      (pseudo_pred << 1) | prev_iter_ends_pseudo_pred;
+  prev_iter_ends_pseudo_pred = pseudo_pred >> 63;
+  uint64_t pseudo_structurals =
+      shifted_pseudo_pred & (~whitespace) & (~quote_mask);
+  structurals |= pseudo_structurals;
+
+  // now, we've used our close quotes all we need to. So let's switch them off
+  // they will be off in the quote mask and on in quote bits.
+  structurals &= ~(quote_bits & ~quote_mask);
+  return structurals;
+}
+
+template <Architecture T = Architecture::NATIVE>
+int find_structural_bits(const uint8_t *buf, size_t len,
+                         simdjson::ParsedJson &pj);
+
+template <Architecture T = Architecture::NATIVE>
+int find_structural_bits(const char *buf, size_t len,
+                         simdjson::ParsedJson &pj) {
+  return find_structural_bits((const uint8_t *)buf, len, pj);
+}
+
+} // namespace simdjson
 
 #endif

include/simdjson/stage1_find_marks_arm64.h

@@ -0,0 +1,210 @@
+#ifndef SIMDJSON_STAGE1_FIND_MARKS_ARM64_H
+#define SIMDJSON_STAGE1_FIND_MARKS_ARM64_H
+
+#include "simdjson/simdutf8check_arm64.h"
+#include "simdjson/stage1_find_marks.h"
+#include "simdjson/stage1_find_marks_flatten.h"
+#include "simdjson/stage1_find_marks_macros.h"
+
+#ifdef IS_ARM64
+namespace simdjson {
+template <> struct simd_input<Architecture::ARM64> {
+  uint8x16_t i0;
+  uint8x16_t i1;
+  uint8x16_t i2;
+  uint8x16_t i3;
+};
+
+template <>
+really_inline simd_input<Architecture::ARM64>
+fill_input<Architecture::ARM64>(const uint8_t *ptr) {
+  struct simd_input<Architecture::ARM64> in;
+  in.i0 = vld1q_u8(ptr + 0);
+  in.i1 = vld1q_u8(ptr + 16);
+  in.i2 = vld1q_u8(ptr + 32);
+  in.i3 = vld1q_u8(ptr + 48);
+  return in;
+}
+
+really_inline uint16_t neon_movemask(uint8x16_t input) {
+  const uint8x16_t bit_mask = {0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
+                               0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80};
+  uint8x16_t minput = vandq_u8(input, bit_mask);
+  uint8x16_t tmp = vpaddq_u8(minput, minput);
+  tmp = vpaddq_u8(tmp, tmp);
+  tmp = vpaddq_u8(tmp, tmp);
+  return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0);
+}
+
+really_inline uint64_t neon_movemask_bulk(uint8x16_t p0, uint8x16_t p1,
+                                          uint8x16_t p2, uint8x16_t p3) {
+  const uint8x16_t bit_mask = {0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
+                               0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80};
+  uint8x16_t t0 = vandq_u8(p0, bit_mask);
+  uint8x16_t t1 = vandq_u8(p1, bit_mask);
+  uint8x16_t t2 = vandq_u8(p2, bit_mask);
+  uint8x16_t t3 = vandq_u8(p3, bit_mask);
+  uint8x16_t sum0 = vpaddq_u8(t0, t1);
+  uint8x16_t sum1 = vpaddq_u8(t2, t3);
+  sum0 = vpaddq_u8(sum0, sum1);
+  sum0 = vpaddq_u8(sum0, sum0);
+  return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0);
+}
+
+template <>
+really_inline uint64_t
+compute_quote_mask<Architecture::ARM64>(uint64_t quote_bits) {
+#ifdef __ARM_FEATURE_CRYPTO // some ARM processors lack this extension
+  return vmull_p64(-1ULL, quote_bits);
+#else
+  return portable_compute_quote_mask(quote_bits);
+#endif
+}
+
+template <> struct utf8_checking_state<Architecture::ARM64> {
+  int8x16_t has_error{};
+  processed_utf_bytes previous{};
+};
+
+// Checks that all bytes are ascii
+really_inline bool check_ascii_neon(simd_input<Architecture::ARM64> in) {
+  // checking if the most significant bit is always equal to 0.
+  uint8x16_t high_bit = vdupq_n_u8(0x80);
+  uint8x16_t t0 = vorrq_u8(in.i0, in.i1);
+  uint8x16_t t1 = vorrq_u8(in.i2, in.i3);
+  uint8x16_t t3 = vorrq_u8(t0, t1);
+  uint8x16_t t4 = vandq_u8(t3, high_bit);
+  uint64x2_t v64 = vreinterpretq_u64_u8(t4);
+  uint32x2_t v32 = vqmovn_u64(v64);
+  uint64x1_t result = vreinterpret_u64_u32(v32);
+  return vget_lane_u64(result, 0) == 0;
+}
+
+template <>
+really_inline void check_utf8<Architecture::ARM64>(
+    simd_input<Architecture::ARM64> in,
+    utf8_checking_state<Architecture::ARM64> &state) {
+  if (check_ascii_neon(in)) {
+    // All bytes are ascii. Therefore the byte that was just before must be
+    // ascii too. We only check the byte that was just before simd_input. Nines
+    // are arbitrary values.
+    const int8x16_t verror =
+        (int8x16_t){9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1};
+    state.has_error =
+        vorrq_s8(vreinterpretq_s8_u8(
+                     vcgtq_s8(state.previous.carried_continuations, verror)),
+                 state.has_error);
+  } else {
+    // it is not ascii so we have to do heavy work
+    state.previous = check_utf8_bytes(vreinterpretq_s8_u8(in.i0),
+                                      &(state.previous), &(state.has_error));
+    state.previous = check_utf8_bytes(vreinterpretq_s8_u8(in.i1),
+                                      &(state.previous), &(state.has_error));
+    state.previous = check_utf8_bytes(vreinterpretq_s8_u8(in.i2),
+                                      &(state.previous), &(state.has_error));
+    state.previous = check_utf8_bytes(vreinterpretq_s8_u8(in.i3),
+                                      &(state.previous), &(state.has_error));
+  }
+}
+
+template <>
+really_inline ErrorValues check_utf8_errors<Architecture::ARM64>(
+    utf8_checking_state<Architecture::ARM64> &state) {
+  uint64x2_t v64 = vreinterpretq_u64_s8(state.has_error);
+  uint32x2_t v32 = vqmovn_u64(v64);
+  uint64x1_t result = vreinterpret_u64_u32(v32);
+  return vget_lane_u64(result, 0) != 0 ? simdjson::UTF8_ERROR
+                                       : simdjson::SUCCESS;
+}
+
+template <>
+really_inline uint64_t cmp_mask_against_input<Architecture::ARM64>(
+    simd_input<Architecture::ARM64> in, uint8_t m) {
+  const uint8x16_t mask = vmovq_n_u8(m);
+  uint8x16_t cmp_res_0 = vceqq_u8(in.i0, mask);
+  uint8x16_t cmp_res_1 = vceqq_u8(in.i1, mask);
+  uint8x16_t cmp_res_2 = vceqq_u8(in.i2, mask);
+  uint8x16_t cmp_res_3 = vceqq_u8(in.i3, mask);
+  return neon_movemask_bulk(cmp_res_0, cmp_res_1, cmp_res_2, cmp_res_3);
+}
+
+template <>
+really_inline uint64_t unsigned_lteq_against_input<Architecture::ARM64>(
+    simd_input<Architecture::ARM64> in, uint8_t m) {
+  const uint8x16_t mask = vmovq_n_u8(m);
+  uint8x16_t cmp_res_0 = vcleq_u8(in.i0, mask);
+  uint8x16_t cmp_res_1 = vcleq_u8(in.i1, mask);
+  uint8x16_t cmp_res_2 = vcleq_u8(in.i2, mask);
+  uint8x16_t cmp_res_3 = vcleq_u8(in.i3, mask);
+  return neon_movemask_bulk(cmp_res_0, cmp_res_1, cmp_res_2, cmp_res_3);
+}
+
+template <>
+really_inline uint64_t find_odd_backslash_sequences<Architecture::ARM64>(
+    simd_input<Architecture::ARM64> in,
+    uint64_t &prev_iter_ends_odd_backslash) {
+  FIND_ODD_BACKSLASH_SEQUENCES(Architecture::ARM64, in,
+                               prev_iter_ends_odd_backslash);
+}
+
+template <>
+really_inline uint64_t find_quote_mask_and_bits<Architecture::ARM64>(
+    simd_input<Architecture::ARM64> in, uint64_t odd_ends,
+    uint64_t &prev_iter_inside_quote, uint64_t &quote_bits,
+    uint64_t &error_mask) {
+  FIND_QUOTE_MASK_AND_BITS(Architecture::ARM64, in, odd_ends,
+                           prev_iter_inside_quote, quote_bits, error_mask)
+}
+
+template <>
+really_inline void find_whitespace_and_structurals<Architecture::ARM64>(
+    simd_input<Architecture::ARM64> in, uint64_t &whitespace,
+    uint64_t &structurals) {
+  const uint8x16_t low_nibble_mask =
+      (uint8x16_t){16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0};
+  const uint8x16_t high_nibble_mask =
+      (uint8x16_t){8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0};
+  const uint8x16_t structural_shufti_mask = vmovq_n_u8(0x7);
+  const uint8x16_t whitespace_shufti_mask = vmovq_n_u8(0x18);
+  const uint8x16_t low_nib_and_mask = vmovq_n_u8(0xf);
+
+  uint8x16_t nib_0_lo = vandq_u8(in.i0, low_nib_and_mask);
+  uint8x16_t nib_0_hi = vshrq_n_u8(in.i0, 4);
+  uint8x16_t shuf_0_lo = vqtbl1q_u8(low_nibble_mask, nib_0_lo);
+  uint8x16_t shuf_0_hi = vqtbl1q_u8(high_nibble_mask, nib_0_hi);
+  uint8x16_t v_0 = vandq_u8(shuf_0_lo, shuf_0_hi);
+
+  uint8x16_t nib_1_lo = vandq_u8(in.i1, low_nib_and_mask);
+  uint8x16_t nib_1_hi = vshrq_n_u8(in.i1, 4);
+  uint8x16_t shuf_1_lo = vqtbl1q_u8(low_nibble_mask, nib_1_lo);
+  uint8x16_t shuf_1_hi = vqtbl1q_u8(high_nibble_mask, nib_1_hi);
+  uint8x16_t v_1 = vandq_u8(shuf_1_lo, shuf_1_hi);
+
+  uint8x16_t nib_2_lo = vandq_u8(in.i2, low_nib_and_mask);
+  uint8x16_t nib_2_hi = vshrq_n_u8(in.i2, 4);
+  uint8x16_t shuf_2_lo = vqtbl1q_u8(low_nibble_mask, nib_2_lo);
+  uint8x16_t shuf_2_hi = vqtbl1q_u8(high_nibble_mask, nib_2_hi);
+  uint8x16_t v_2 = vandq_u8(shuf_2_lo, shuf_2_hi);
+
+  uint8x16_t nib_3_lo = vandq_u8(in.i3, low_nib_and_mask);
+  uint8x16_t nib_3_hi = vshrq_n_u8(in.i3, 4);
+  uint8x16_t shuf_3_lo = vqtbl1q_u8(low_nibble_mask, nib_3_lo);
+  uint8x16_t shuf_3_hi = vqtbl1q_u8(high_nibble_mask, nib_3_hi);
+  uint8x16_t v_3 = vandq_u8(shuf_3_lo, shuf_3_hi);
+
+  uint8x16_t tmp_0 = vtstq_u8(v_0, structural_shufti_mask);
+  uint8x16_t tmp_1 = vtstq_u8(v_1, structural_shufti_mask);
+  uint8x16_t tmp_2 = vtstq_u8(v_2, structural_shufti_mask);
+  uint8x16_t tmp_3 = vtstq_u8(v_3, structural_shufti_mask);
+  structurals = neon_movemask_bulk(tmp_0, tmp_1, tmp_2, tmp_3);
+
+  uint8x16_t tmp_ws_0 = vtstq_u8(v_0, whitespace_shufti_mask);
+  uint8x16_t tmp_ws_1 = vtstq_u8(v_1, whitespace_shufti_mask);
+  uint8x16_t tmp_ws_2 = vtstq_u8(v_2, whitespace_shufti_mask);
+  uint8x16_t tmp_ws_3 = vtstq_u8(v_3, whitespace_shufti_mask);
+  whitespace = neon_movemask_bulk(tmp_ws_0, tmp_ws_1, tmp_ws_2, tmp_ws_3);
+}
+} // namespace simdjson
+
+#endif // IS_ARM64
+#endif // SIMDJSON_STAGE1_FIND_MARKS_ARM64_H

include/simdjson/stage1_find_marks_flatten.h

@@ -0,0 +1,93 @@
+#ifndef SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_H
+#define SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_H
+
+namespace simdjson {
+
+#ifdef SIMDJSON_NAIVE_FLATTEN // useful for benchmarking
+//
+// This is just a naive implementation. It should be normally
+// disable, but can be used for research purposes to compare
+// again our optimized version.
+really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base,
+                                uint32_t idx, uint64_t bits) {
+  uint32_t *out_ptr = base_ptr + base;
+  idx -= 64;
+  while (bits != 0) {
+    out_ptr[0] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    out_ptr++;
+  }
+  base = (out_ptr - base_ptr);
+}
+
+#else
+// flatten out values in 'bits' assuming that they are are to have values of idx
+// plus their position in the bitvector, and store these indexes at
+// base_ptr[base] incrementing base as we go
+// will potentially store extra values beyond end of valid bits, so base_ptr
+// needs to be large enough to handle this
+really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base,
+                                uint32_t idx, uint64_t bits) {
+  // In some instances, the next branch is expensive because it is mispredicted.
+  // Unfortunately, in other cases,
+  // it helps tremendously.
+  if (bits == 0)
+    return;
+  uint32_t cnt = hamming(bits);
+  uint32_t next_base = base + cnt;
+  idx -= 64;
+  base_ptr += base;
+  {
+    base_ptr[0] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[1] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[2] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[3] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[4] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[5] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[6] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[7] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr += 8;
+  }
+  // We hope that the next branch is easily predicted.
+  if (cnt > 8) {
+    base_ptr[0] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[1] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[2] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[3] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[4] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[5] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[6] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[7] = idx + trailing_zeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr += 8;
+  }
+  if (cnt > 16) { // unluckly: we rarely get here
+    // since it means having one structural or pseudo-structral element
+    // every 4 characters (possible with inputs like "","","",...).
+    do {
+      base_ptr[0] = idx + trailing_zeroes(bits);
+      bits = bits & (bits - 1);
+      base_ptr++;
+    } while (bits != 0);
+  }
+  base = next_base;
+}
+#endif // SIMDJSON_NAIVE_FLATTEN
+} // namespace simdjson
+
+#endif // SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_H

include/simdjson/stage1_find_marks_flatten_haswell.h

@@ -0,0 +1,95 @@
+#ifndef SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_HASWELL_H
+#define SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_HASWELL_H
+
+// This file provides the same function as
+// stage1_find_marks_flatten.h, but uses Intel intrinsics.
+// This should provide better performance on Visual Studio
+// and other compilers that do a conservative optimization.
+
+// Specifically, on x64 processors with BMI,
+// x & (x - 1) should be mapped to
+// the blsr instruction. By using the
+// _blsr_u64 intrinsic, we
+// ensure that this will happen.
+/////////
+
+#include "simdjson/common_defs.h"
+#include "simdjson/portability.h"
+
+#ifdef IS_X86_64
+
+TARGET_HASWELL
+namespace simdjson {
+namespace haswell {
+
+// flatten out values in 'bits' assuming that they are are to have values of idx
+// plus their position in the bitvector, and store these indexes at
+// base_ptr[base] incrementing base as we go
+// will potentially store extra values beyond end of valid bits, so base_ptr
+// needs to be large enough to handle this
+really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base,
+                                uint32_t idx, uint64_t bits) {
+  // In some instances, the next branch is expensive because it is mispredicted.
+  // Unfortunately, in other cases,
+  // it helps tremendously.
+  if (bits == 0)
+    return;
+  uint32_t cnt = _mm_popcnt_u64(bits);
+  uint32_t next_base = base + cnt;
+  idx -= 64;
+  base_ptr += base;
+  {
+    base_ptr[0] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[1] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[2] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[3] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[4] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[5] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[6] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[7] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr += 8;
+  }
+  // We hope that the next branch is easily predicted.
+  if (cnt > 8) {
+    base_ptr[0] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[1] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[2] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[3] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[4] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[5] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[6] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr[7] = idx + trailing_zeroes(bits);
+    bits = _blsr_u64(bits);
+    base_ptr += 8;
+  }
+  if (cnt > 16) { // unluckly: we rarely get here
+    // since it means having one structural or pseudo-structral element
+    // every 4 characters (possible with inputs like "","","",...).
+    do {
+      base_ptr[0] = idx + trailing_zeroes(bits);
+      bits = _blsr_u64(bits);
+      base_ptr++;
+    } while (bits != 0);
+  }
+  base = next_base;
+}
+} // namespace haswell
+} // namespace simdjson
+UNTARGET_REGION
+#endif // IS_X86_64
+#endif // SIMDJSON_STAGE1_FIND_MARKS_FLATTEN_H

include/simdjson/stage1_find_marks_haswell.h

@@ -0,0 +1,210 @@
+#ifndef SIMDJSON_STAGE1_FIND_MARKS_HASWELL_H
+#define SIMDJSON_STAGE1_FIND_MARKS_HASWELL_H
+
+#include "simdjson/simdutf8check_haswell.h"
+#include "simdjson/stage1_find_marks.h"
+#include "simdjson/stage1_find_marks_flatten_haswell.h"
+#include "simdjson/stage1_find_marks_macros.h"
+
+#ifdef IS_X86_64
+
+TARGET_HASWELL
+namespace simdjson {
+template <> struct simd_input<Architecture::HASWELL> {
+  __m256i lo;
+  __m256i hi;
+};
+
+template <>
+really_inline simd_input<Architecture::HASWELL>
+fill_input<Architecture::HASWELL>(const uint8_t *ptr) {
+  struct simd_input<Architecture::HASWELL> in;
+  in.lo = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(ptr + 0));
+  in.hi = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(ptr + 32));
+  return in;
+}
+
+template <>
+really_inline uint64_t
+compute_quote_mask<Architecture::HASWELL>(uint64_t quote_bits) {
+  // There should be no such thing with a processing supporting avx2
+  // but not clmul.
+  uint64_t quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128(
+      _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFFu), 0));
+  return quote_mask;
+}
+
+template <> struct utf8_checking_state<Architecture::HASWELL> {
+  __m256i has_error;
+  avx_processed_utf_bytes previous;
+  utf8_checking_state() {
+    has_error = _mm256_setzero_si256();
+    previous.raw_bytes = _mm256_setzero_si256();
+    previous.high_nibbles = _mm256_setzero_si256();
+    previous.carried_continuations = _mm256_setzero_si256();
+  }
+};
+
+template <>
+really_inline void check_utf8<Architecture::HASWELL>(
+    simd_input<Architecture::HASWELL> in,
+    utf8_checking_state<Architecture::HASWELL> &state) {
+  __m256i high_bit = _mm256_set1_epi8(0x80u);
+  if ((_mm256_testz_si256(_mm256_or_si256(in.lo, in.hi), high_bit)) == 1) {
+    // it is ascii, we just check continuation
+    state.has_error = _mm256_or_si256(
+        _mm256_cmpgt_epi8(state.previous.carried_continuations,
+                          _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+                                           9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+                                           9, 9, 9, 9, 9, 9, 9, 1)),
+        state.has_error);
+  } else {
+    // it is not ascii so we have to do heavy work
+    state.previous =
+        avx_check_utf8_bytes(in.lo, &(state.previous), &(state.has_error));
+    state.previous =
+        avx_check_utf8_bytes(in.hi, &(state.previous), &(state.has_error));
+  }
+}
+
+template <>
+really_inline ErrorValues check_utf8_errors<Architecture::HASWELL>(
+    utf8_checking_state<Architecture::HASWELL> &state) {
+  return _mm256_testz_si256(state.has_error, state.has_error) == 0
+             ? simdjson::UTF8_ERROR
+             : simdjson::SUCCESS;
+}
+
+template <>
+really_inline uint64_t cmp_mask_against_input<Architecture::HASWELL>(
+    simd_input<Architecture::HASWELL> in, uint8_t m) {
+  const __m256i mask = _mm256_set1_epi8(m);
+  __m256i cmp_res_0 = _mm256_cmpeq_epi8(in.lo, mask);
+  uint64_t res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(cmp_res_0));
+  __m256i cmp_res_1 = _mm256_cmpeq_epi8(in.hi, mask);
+  uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1);
+  return res_0 | (res_1 << 32);
+}
+
+template <>
+really_inline uint64_t unsigned_lteq_against_input<Architecture::HASWELL>(
+    simd_input<Architecture::HASWELL> in, uint8_t m) {
+  const __m256i maxval = _mm256_set1_epi8(m);
+  __m256i cmp_res_0 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, in.lo), maxval);
+  uint64_t res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(cmp_res_0));
+  __m256i cmp_res_1 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval, in.hi), maxval);
+  uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1);
+  return res_0 | (res_1 << 32);
+}
+
+template <>
+really_inline uint64_t find_odd_backslash_sequences<Architecture::HASWELL>(
+    simd_input<Architecture::HASWELL> in,
+    uint64_t &prev_iter_ends_odd_backslash) {
+  FIND_ODD_BACKSLASH_SEQUENCES(Architecture::HASWELL, in,
+                               prev_iter_ends_odd_backslash);
+}
+
+template <>
+really_inline uint64_t find_quote_mask_and_bits<Architecture::HASWELL>(
+    simd_input<Architecture::HASWELL> in, uint64_t odd_ends,
+    uint64_t &prev_iter_inside_quote, uint64_t &quote_bits,
+    uint64_t &error_mask) {
+  FIND_QUOTE_MASK_AND_BITS(Architecture::HASWELL, in, odd_ends,
+                           prev_iter_inside_quote, quote_bits, error_mask)
+}
+
+template <>
+really_inline void find_whitespace_and_structurals<Architecture::HASWELL>(
+    simd_input<Architecture::HASWELL> in, uint64_t &whitespace,
+    uint64_t &structurals) {
+#ifdef SIMDJSON_NAIVE_STRUCTURAL
+  // You should never need this naive approach, but it can be useful
+  // for research purposes
+  const __m256i mask_open_brace = _mm256_set1_epi8(0x7b);
+  __m256i struct_lo = _mm256_cmpeq_epi8(in.lo, mask_open_brace);
+  __m256i struct_hi = _mm256_cmpeq_epi8(in.hi, mask_open_brace);
+  const __m256i mask_close_brace = _mm256_set1_epi8(0x7d);
+  struct_lo =
+      _mm256_or_si256(struct_lo, _mm256_cmpeq_epi8(in.lo, mask_close_brace));
+  struct_hi =
+      _mm256_or_si256(struct_hi, _mm256_cmpeq_epi8(in.hi, mask_close_brace));
+  const __m256i mask_open_bracket = _mm256_set1_epi8(0x5b);
+  struct_lo =
+      _mm256_or_si256(struct_lo, _mm256_cmpeq_epi8(in.lo, mask_open_bracket));
+  struct_hi =
+      _mm256_or_si256(struct_hi, _mm256_cmpeq_epi8(in.hi, mask_open_bracket));
+  const __m256i mask_close_bracket = _mm256_set1_epi8(0x5d);
+  struct_lo =
+      _mm256_or_si256(struct_lo, _mm256_cmpeq_epi8(in.lo, mask_close_bracket));
+  struct_hi =
+      _mm256_or_si256(struct_hi, _mm256_cmpeq_epi8(in.hi, mask_close_bracket));
+  const __m256i mask_column = _mm256_set1_epi8(0x3a);
+  struct_lo = _mm256_or_si256(struct_lo, _mm256_cmpeq_epi8(in.lo, mask_column));
+  struct_hi = _mm256_or_si256(struct_hi, _mm256_cmpeq_epi8(in.hi, mask_column));
+  const __m256i mask_comma = _mm256_set1_epi8(0x2c);
+  struct_lo = _mm256_or_si256(struct_lo, _mm256_cmpeq_epi8(in.lo, mask_comma));
+  struct_hi = _mm256_or_si256(struct_hi, _mm256_cmpeq_epi8(in.hi, mask_comma));
+  uint64_t structural_res_0 =
+      static_cast<uint32_t>(_mm256_movemask_epi8(struct_lo));
+  uint64_t structural_res_1 = _mm256_movemask_epi8(struct_hi);
+  structurals = (structural_res_0 | (structural_res_1 << 32));
+
+  const __m256i mask_space = _mm256_set1_epi8(0x20);
+  __m256i space_lo = _mm256_cmpeq_epi8(in.lo, mask_space);
+  __m256i space_hi = _mm256_cmpeq_epi8(in.hi, mask_space);
+  const __m256i mask_linefeed = _mm256_set1_epi8(0x0a);
+  space_lo = _mm256_or_si256(space_lo, _mm256_cmpeq_epi8(in.lo, mask_linefeed));
+  space_hi = _mm256_or_si256(space_hi, _mm256_cmpeq_epi8(in.hi, mask_linefeed));
+  const __m256i mask_tab = _mm256_set1_epi8(0x09);
+  space_lo = _mm256_or_si256(space_lo, _mm256_cmpeq_epi8(in.lo, mask_tab));
+  space_hi = _mm256_or_si256(space_hi, _mm256_cmpeq_epi8(in.hi, mask_tab));
+  const __m256i mask_carriage = _mm256_set1_epi8(0x0d);
+  space_lo = _mm256_or_si256(space_lo, _mm256_cmpeq_epi8(in.lo, mask_carriage));
+  space_hi = _mm256_or_si256(space_hi, _mm256_cmpeq_epi8(in.hi, mask_carriage));
+
+  uint64_t ws_res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(space_lo));
+  uint64_t ws_res_1 = _mm256_movemask_epi8(space_hi);
+  whitespace = (ws_res_0 | (ws_res_1 << 32));
+  // end of naive approach
+
+#else  // SIMDJSON_NAIVE_STRUCTURAL
+  // clang-format off
+  const __m256i structural_table =
+      _mm256_setr_epi8(44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123,
+                       44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123);
+  const __m256i white_table = _mm256_setr_epi8(
+      32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100,
+      32, 100, 100, 100, 17, 100, 113, 2, 100, 9, 10, 112, 100, 13, 100, 100);
+  // clang-format on
+  const __m256i struct_offset = _mm256_set1_epi8(0xd4u);
+  const __m256i struct_mask = _mm256_set1_epi8(32);
+
+  __m256i lo_white =
+      _mm256_cmpeq_epi8(in.lo, _mm256_shuffle_epi8(white_table, in.lo));
+  __m256i hi_white =
+      _mm256_cmpeq_epi8(in.hi, _mm256_shuffle_epi8(white_table, in.hi));
+  uint64_t ws_res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(lo_white));
+  uint64_t ws_res_1 = _mm256_movemask_epi8(hi_white);
+  whitespace = (ws_res_0 | (ws_res_1 << 32));
+  __m256i lo_struct_r1 = _mm256_add_epi8(struct_offset, in.lo);
+  __m256i hi_struct_r1 = _mm256_add_epi8(struct_offset, in.hi);
+  __m256i lo_struct_r2 = _mm256_or_si256(in.lo, struct_mask);
+  __m256i hi_struct_r2 = _mm256_or_si256(in.hi, struct_mask);
+  __m256i lo_struct_r3 = _mm256_shuffle_epi8(structural_table, lo_struct_r1);
+  __m256i hi_struct_r3 = _mm256_shuffle_epi8(structural_table, hi_struct_r1);
+  __m256i lo_struct = _mm256_cmpeq_epi8(lo_struct_r2, lo_struct_r3);
+  __m256i hi_struct = _mm256_cmpeq_epi8(hi_struct_r2, hi_struct_r3);
+
+  uint64_t structural_res_0 =
+      static_cast<uint32_t>(_mm256_movemask_epi8(lo_struct));
+  uint64_t structural_res_1 = _mm256_movemask_epi8(hi_struct);
+  structurals = (structural_res_0 | (structural_res_1 << 32));
+#endif // SIMDJSON_NAIVE_STRUCTURAL
+}
+
+} // namespace simdjson
+UNTARGET_REGION
+
+#endif // IS_X86_64
+#endif // SIMDJSON_STAGE1_FIND_MARKS_HASWELL_H

include/simdjson/stage1_find_marks_macros.h

@@ -0,0 +1,239 @@
+#ifndef SIMDJSON_STAGE1_FIND_MARKS_MACROS_H
+#define SIMDJSON_STAGE1_FIND_MARKS_MACROS_H
+
+// return a bitvector indicating where we have characters that end an odd-length
+// sequence of backslashes (and thus change the behavior of the next character
+// to follow). A even-length sequence of backslashes, and, for that matter, the
+// largest even-length prefix of our odd-length sequence of backslashes, simply
+// modify the behavior of the backslashes themselves.
+// We also update the prev_iter_ends_odd_backslash reference parameter to
+// indicate whether we end an iteration on an odd-length sequence of
+// backslashes, which modifies our subsequent search for odd-length
+// sequences of backslashes in an obvious way.
+// We need to compile that code for multiple architectures. However, target
+// attributes can be used only once by function definition. Huge macro seemed
+// better than huge code duplication. uint64_t
+// FIND_ODD_BACKSLASH_SEQUENCES(Architecture T, simd_input<T> in, uint64_t
+// &prev_iter_ends_odd_backslash)
+#define FIND_ODD_BACKSLASH_SEQUENCES(T, in, prev_iter_ends_odd_backslash)      \
+  {                                                                            \
+    const uint64_t even_bits = 0x5555555555555555ULL;                          \
+    const uint64_t odd_bits = ~even_bits;                                      \
+    uint64_t bs_bits = cmp_mask_against_input<T>(in, '\\');                    \
+    uint64_t start_edges = bs_bits & ~(bs_bits << 1);                          \
+    /* flip lowest if we have an odd-length run at the end of the prior        \
+     * iteration */                                                            \
+    uint64_t even_start_mask = even_bits ^ prev_iter_ends_odd_backslash;       \
+    uint64_t even_starts = start_edges & even_start_mask;                      \
+    uint64_t odd_starts = start_edges & ~even_start_mask;                      \
+    uint64_t even_carries = bs_bits + even_starts;                             \
+                                                                               \
+    uint64_t odd_carries;                                                      \
+    /* must record the carry-out of our odd-carries out of bit 63; this        \
+     * indicates whether the sense of any edge going to the next iteration     \
+     * should be flipped */                                                    \
+    bool iter_ends_odd_backslash =                                             \
+        add_overflow(bs_bits, odd_starts, &odd_carries);                       \
+                                                                               \
+    odd_carries |= prev_iter_ends_odd_backslash; /* push in bit zero as a      \
+                                                  * potential end if we had an \
+                                                  * odd-numbered run at the    \
+                                                  * end of the previous        \
+                                                  * iteration */               \
+    prev_iter_ends_odd_backslash = iter_ends_odd_backslash ? 0x1ULL : 0x0ULL;  \
+    uint64_t even_carry_ends = even_carries & ~bs_bits;                        \
+    uint64_t odd_carry_ends = odd_carries & ~bs_bits;                          \
+    uint64_t even_start_odd_end = even_carry_ends & odd_bits;                  \
+    uint64_t odd_start_even_end = odd_carry_ends & even_bits;                  \
+    uint64_t odd_ends = even_start_odd_end | odd_start_even_end;               \
+    return odd_ends;                                                           \
+  }
+
+// return both the quote mask (which is a half-open mask that covers the first
+// quote
+// in an unescaped quote pair and everything in the quote pair) and the quote
+// bits, which are the simple
+// unescaped quoted bits. We also update the prev_iter_inside_quote value to
+// tell the next iteration
+// whether we finished the final iteration inside a quote pair; if so, this
+// inverts our behavior of
+// whether we're inside quotes for the next iteration.
+// Note that we don't do any error checking to see if we have backslash
+// sequences outside quotes; these
+// backslash sequences (of any length) will be detected elsewhere.
+// We need to compile that code for multiple architectures. However, target
+// attributes can be used only once by function definition. Huge macro seemed
+// better than huge code duplication. uint64_t
+// FIND_QUOTE_MASK_AND_BITS(Architecture T, simd_input<T> in, uint64_t odd_ends,
+//    uint64_t &prev_iter_inside_quote, uint64_t &quote_bits, uint64_t
+//    &error_mask)
+#define FIND_QUOTE_MASK_AND_BITS(T, in, odd_ends, prev_iter_inside_quote,      \
+                                 quote_bits, error_mask)                       \
+  {                                                                            \
+    quote_bits = cmp_mask_against_input<T>(in, '"');                           \
+    quote_bits = quote_bits & ~odd_ends;                                       \
+    uint64_t quote_mask = compute_quote_mask<T>(quote_bits);                   \
+    quote_mask ^= prev_iter_inside_quote;                                      \
+    /* All Unicode characters may be placed within the                         \
+     * quotation marks, except for the characters that MUST be escaped:        \
+     * quotation mark, reverse solidus, and the control characters (U+0000     \
+     * through U+001F).                                                        \
+     * https://tools.ietf.org/html/rfc8259 */                                  \
+    uint64_t unescaped = unsigned_lteq_against_input<T>(in, 0x1F);             \
+    error_mask |= quote_mask & unescaped;                                      \
+    /* right shift of a signed value expected to be well-defined and standard  \
+     * compliant as of C++20,                                                  \
+     * John Regher from Utah U. says this is fine code */                      \
+    prev_iter_inside_quote =                                                   \
+        static_cast<uint64_t>(static_cast<int64_t>(quote_mask) >> 63);         \
+    return quote_mask;                                                         \
+  }
+
+// Find structural bits in a 64-byte chunk.
+// We need to compile that code for multiple architectures. However, target
+// attributes can be used only once by function definition. Huge macro seemed
+// better than huge code duplication. void FIND_STRUCTURAL_BITS_64(
+//                              Architecture T,
+//                              const uint8_t *buf,
+//                              size_t idx,
+//                              uint32_t *base_ptr,
+//                              uint32_t &base,
+//                              uint64_t &prev_iter_ends_odd_backslash,
+//                              uint64_t &prev_iter_inside_quote,
+//                              uint64_t &prev_iter_ends_pseudo_pred,
+//                              uint64_t &structurals,
+//                              uint64_t &error_mask,
+//                              utf8_checking_state<T> &utf8_state, flatten
+//                              function)
+#define FIND_STRUCTURAL_BITS_64(                                               \
+    T, buf, idx, base_ptr, base, prev_iter_ends_odd_backslash,                 \
+    prev_iter_inside_quote, prev_iter_ends_pseudo_pred, structurals,           \
+    error_mask, utf8_state, flat)                                              \
+  {                                                                            \
+    simd_input<T> in = fill_input<T>(buf);                                     \
+    check_utf8<T>(in, utf8_state);                                             \
+    /* detect odd sequences of backslashes */                                  \
+    uint64_t odd_ends =                                                        \
+        find_odd_backslash_sequences<T>(in, prev_iter_ends_odd_backslash);     \
+                                                                               \
+    /* detect insides of quote pairs ("quote_mask") and also our quote_bits    \
+     * themselves */                                                           \
+    uint64_t quote_bits;                                                       \
+    uint64_t quote_mask = find_quote_mask_and_bits<T>(                         \
+        in, odd_ends, prev_iter_inside_quote, quote_bits, error_mask);         \
+                                                                               \
+    /* take the previous iterations structural bits, not our current           \
+     * iteration,                                                              \
+     * and flatten */                                                          \
+    flat(base_ptr, base, idx, structurals);                                    \
+                                                                               \
+    uint64_t whitespace;                                                       \
+    find_whitespace_and_structurals<T>(in, whitespace, structurals);           \
+                                                                               \
+    /* fixup structurals to reflect quotes and add pseudo-structural           \
+     * characters */                                                           \
+    structurals =                                                              \
+        finalize_structurals(structurals, whitespace, quote_mask, quote_bits,  \
+                             prev_iter_ends_pseudo_pred);                      \
+  }
+
+// We need to compile that code for multiple architectures. However, target
+// attributes can be used only once by function definition. Huge macro seemed
+// better than huge code duplication. ErrorValues
+// FIND_STRUCTURAL_BITS(Architecture T, const uint8_t *buf, size_t len,
+// ParsedJson &pj, flatten function)
+#define FIND_STRUCTURAL_BITS(T, buf, len, pj, flat)                            \
+  {                                                                            \
+    if (len > pj.byte_capacity) {                                              \
+      std::cerr << "Your ParsedJson object only supports documents up to "     \
+                << pj.byte_capacity << " bytes but you are trying to process " \
+                << len << " bytes" << std::endl;                               \
+      return simdjson::CAPACITY;                                               \
+    }                                                                          \
+    uint32_t *base_ptr = pj.structural_indexes;                                \
+    uint32_t base = 0;                                                         \
+    utf8_checking_state<T> utf8_state;                                         \
+                                                                               \
+    /* we have padded the input out to 64 byte multiple with the remainder     \
+     * being zeros persistent state across loop does the last iteration end    \
+     * with an odd-length sequence of backslashes? */                          \
+                                                                               \
+    /* either 0 or 1, but a 64-bit value */                                    \
+    uint64_t prev_iter_ends_odd_backslash = 0ULL;                              \
+    /* does the previous iteration end inside a double-quote pair? */          \
+    uint64_t prev_iter_inside_quote =                                          \
+        0ULL; /* either all zeros or all ones                                  \
+               * does the previous iteration end on something that is a        \
+               * predecessor of a pseudo-structural character - i.e.           \
+               * whitespace or a structural character effectively the very     \
+               * first char is considered to follow "whitespace" for the       \
+               * purposes of pseudo-structural character detection so we       \
+               * initialize to 1 */                                            \
+    uint64_t prev_iter_ends_pseudo_pred = 1ULL;                                \
+                                                                               \
+    /* structurals are persistent state across loop as we flatten them on the  \
+     * subsequent iteration into our array pointed to be base_ptr.             \
+     * This is harmless on the first iteration as structurals==0               \
+     * and is done for performance reasons; we can hide some of the latency of \
+     * the                                                                     \
+     * expensive carryless multiply in the previous step with this work */     \
+    uint64_t structurals = 0;                                                  \
+                                                                               \
+    size_t lenminus64 = len < 64 ? 0 : len - 64;                               \
+    size_t idx = 0;                                                            \
+    uint64_t error_mask = 0; /* for unescaped characters within strings (ASCII \
+                                code points < 0x20) */                         \
+                                                                               \
+    for (; idx < lenminus64; idx += 64) {                                      \
+      FIND_STRUCTURAL_BITS_64(                                                 \
+          T, &buf[idx], idx, base_ptr, base, prev_iter_ends_odd_backslash,     \
+          prev_iter_inside_quote, prev_iter_ends_pseudo_pred, structurals,     \
+          error_mask, utf8_state, flat);                                       \
+    }                                                                          \
+    /* If we have a final chunk of less than 64 bytes, pad it to 64 with       \
+     * spaces  before processing it (otherwise, we risk invalidating the UTF-8 \
+     * checks). */                                                             \
+    if (idx < len) {                                                           \
+      uint8_t tmp_buf[64];                                                     \
+      memset(tmp_buf, 0x20, 64);                                               \
+      memcpy(tmp_buf, buf + idx, len - idx);                                   \
+      FIND_STRUCTURAL_BITS_64(                                                 \
+          T, &tmp_buf[0], idx, base_ptr, base, prev_iter_ends_odd_backslash,   \
+          prev_iter_inside_quote, prev_iter_ends_pseudo_pred, structurals,     \
+          error_mask, utf8_state, flat);                                       \
+      idx += 64;                                                               \
+    }                                                                          \
+                                                                               \
+    /* is last string quote closed? */                                         \
+    if (prev_iter_inside_quote) {                                              \
+      return simdjson::UNCLOSED_STRING;                                        \
+    }                                                                          \
+                                                                               \
+    /* finally, flatten out the remaining structurals from the last iteration  \
+     */                                                                        \
+    flat(base_ptr, base, idx, structurals);                                    \
+                                                                               \
+    pj.n_structural_indexes = base;                                            \
+    /* a valid JSON file cannot have zero structural indexes - we should have  \
+     * found something */                                                      \
+    if (pj.n_structural_indexes == 0u) {                                       \
+      return simdjson::EMPTY;                                                  \
+    }                                                                          \
+    if (base_ptr[pj.n_structural_indexes - 1] > len) {                         \
+      return simdjson::UNEXPECTED_ERROR;                                       \
+    }                                                                          \
+    if (len != base_ptr[pj.n_structural_indexes - 1]) {                        \
+      /* the string might not be NULL terminated, but we add a virtual NULL    \
+       * ending                                                                \
+       * character. */                                                         \
+      base_ptr[pj.n_structural_indexes++] = len;                               \
+    }                                                                          \
+    /* make it safe to dereference one beyond this array */                    \
+    base_ptr[pj.n_structural_indexes] = 0;                                     \
+    if (error_mask) {                                                          \
+      return simdjson::UNESCAPED_CHARS;                                        \
+    }                                                                          \
+    return check_utf8_errors<T>(utf8_state);                                   \
+  }
+
+#endif // SIMDJSON_STAGE1_FIND_MARKS_MACROS_H

include/simdjson/stage1_find_marks_westmere.h

@@ -0,0 +1,194 @@
+#ifndef SIMDJSON_STAGE1_FIND_MARKS_WESTMERE_H
+#define SIMDJSON_STAGE1_FIND_MARKS_WESTMERE_H
+
+#include "simdjson/simdutf8check_westmere.h"
+#include "simdjson/stage1_find_marks.h"
+#include "simdjson/stage1_find_marks_flatten.h"
+#include "simdjson/stage1_find_marks_macros.h"
+
+#ifdef IS_X86_64
+
+TARGET_WESTMERE
+namespace simdjson {
+template <> struct simd_input<Architecture::WESTMERE> {
+  __m128i v0;
+  __m128i v1;
+  __m128i v2;
+  __m128i v3;
+};
+
+template <>
+really_inline simd_input<Architecture::WESTMERE>
+fill_input<Architecture::WESTMERE>(const uint8_t *ptr) {
+  struct simd_input<Architecture::WESTMERE> in;
+  in.v0 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(ptr + 0));
+  in.v1 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(ptr + 16));
+  in.v2 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(ptr + 32));
+  in.v3 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(ptr + 48));
+  return in;
+}
+
+template <>
+really_inline uint64_t
+compute_quote_mask<Architecture::WESTMERE>(uint64_t quote_bits) {
+  return _mm_cvtsi128_si64(_mm_clmulepi64_si128(
+      _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFFu), 0));
+}
+
+template <> struct utf8_checking_state<Architecture::WESTMERE> {
+  __m128i has_error = _mm_setzero_si128();
+  processed_utf_bytes previous{
+      _mm_setzero_si128(), // raw_bytes
+      _mm_setzero_si128(), // high_nibbles
+      _mm_setzero_si128()  // carried_continuations
+  };
+};
+
+template <>
+really_inline void check_utf8<Architecture::WESTMERE>(
+    simd_input<Architecture::WESTMERE> in,
+    utf8_checking_state<Architecture::WESTMERE> &state) {
+  __m128i high_bit = _mm_set1_epi8(0x80u);
+  if ((_mm_testz_si128(_mm_or_si128(in.v0, in.v1), high_bit)) == 1) {
+    // it is ascii, we just check continuation
+    state.has_error =
+        _mm_or_si128(_mm_cmpgt_epi8(state.previous.carried_continuations,
+                                    _mm_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+                                                  9, 9, 9, 9, 9, 1)),
+                     state.has_error);
+  } else {
+    // it is not ascii so we have to do heavy work
+    state.previous =
+        check_utf8_bytes(in.v0, &(state.previous), &(state.has_error));
+    state.previous =
+        check_utf8_bytes(in.v1, &(state.previous), &(state.has_error));
+  }
+
+  if ((_mm_testz_si128(_mm_or_si128(in.v2, in.v3), high_bit)) == 1) {
+    // it is ascii, we just check continuation
+    state.has_error =
+        _mm_or_si128(_mm_cmpgt_epi8(state.previous.carried_continuations,
+                                    _mm_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+                                                  9, 9, 9, 9, 9, 1)),
+                     state.has_error);
+  } else {
+    // it is not ascii so we have to do heavy work
+    state.previous =
+        check_utf8_bytes(in.v2, &(state.previous), &(state.has_error));
+    state.previous =
+        check_utf8_bytes(in.v3, &(state.previous), &(state.has_error));
+  }
+}
+
+template <>
+really_inline ErrorValues check_utf8_errors<Architecture::WESTMERE>(
+    utf8_checking_state<Architecture::WESTMERE> &state) {
+  return _mm_testz_si128(state.has_error, state.has_error) == 0
+             ? simdjson::UTF8_ERROR
+             : simdjson::SUCCESS;
+}
+
+template <>
+really_inline uint64_t cmp_mask_against_input<Architecture::WESTMERE>(
+    simd_input<Architecture::WESTMERE> in, uint8_t m) {
+  const __m128i mask = _mm_set1_epi8(m);
+  __m128i cmp_res_0 = _mm_cmpeq_epi8(in.v0, mask);
+  uint64_t res_0 = _mm_movemask_epi8(cmp_res_0);
+  __m128i cmp_res_1 = _mm_cmpeq_epi8(in.v1, mask);
+  uint64_t res_1 = _mm_movemask_epi8(cmp_res_1);
+  __m128i cmp_res_2 = _mm_cmpeq_epi8(in.v2, mask);
+  uint64_t res_2 = _mm_movemask_epi8(cmp_res_2);
+  __m128i cmp_res_3 = _mm_cmpeq_epi8(in.v3, mask);
+  uint64_t res_3 = _mm_movemask_epi8(cmp_res_3);
+  return res_0 | (res_1 << 16) | (res_2 << 32) | (res_3 << 48);
+}
+
+template <>
+really_inline uint64_t unsigned_lteq_against_input<Architecture::WESTMERE>(
+    simd_input<Architecture::WESTMERE> in, uint8_t m) {
+  const __m128i maxval = _mm_set1_epi8(m);
+  __m128i cmp_res_0 = _mm_cmpeq_epi8(_mm_max_epu8(maxval, in.v0), maxval);
+  uint64_t res_0 = _mm_movemask_epi8(cmp_res_0);
+  __m128i cmp_res_1 = _mm_cmpeq_epi8(_mm_max_epu8(maxval, in.v1), maxval);
+  uint64_t res_1 = _mm_movemask_epi8(cmp_res_1);
+  __m128i cmp_res_2 = _mm_cmpeq_epi8(_mm_max_epu8(maxval, in.v2), maxval);
+  uint64_t res_2 = _mm_movemask_epi8(cmp_res_2);
+  __m128i cmp_res_3 = _mm_cmpeq_epi8(_mm_max_epu8(maxval, in.v3), maxval);
+  uint64_t res_3 = _mm_movemask_epi8(cmp_res_3);
+  return res_0 | (res_1 << 16) | (res_2 << 32) | (res_3 << 48);
+}
+
+template <>
+really_inline uint64_t find_odd_backslash_sequences<Architecture::WESTMERE>(
+    simd_input<Architecture::WESTMERE> in,
+    uint64_t &prev_iter_ends_odd_backslash) {
+  FIND_ODD_BACKSLASH_SEQUENCES(Architecture::WESTMERE, in,
+                               prev_iter_ends_odd_backslash);
+}
+
+template <>
+really_inline uint64_t find_quote_mask_and_bits<Architecture::WESTMERE>(
+    simd_input<Architecture::WESTMERE> in, uint64_t odd_ends,
+    uint64_t &prev_iter_inside_quote, uint64_t &quote_bits,
+    uint64_t &error_mask) {
+  FIND_QUOTE_MASK_AND_BITS(Architecture::WESTMERE, in, odd_ends,
+                           prev_iter_inside_quote, quote_bits, error_mask)
+}
+
+template <>
+really_inline void find_whitespace_and_structurals<Architecture::WESTMERE>(
+    simd_input<Architecture::WESTMERE> in, uint64_t &whitespace,
+    uint64_t &structurals) {
+  const __m128i structural_table =
+      _mm_setr_epi8(44, 125, 0, 0, 0xc0u, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 123);
+  const __m128i white_table = _mm_setr_epi8(32, 100, 100, 100, 17, 100, 113, 2,
+                                            100, 9, 10, 112, 100, 13, 100, 100);
+  const __m128i struct_offset = _mm_set1_epi8(0xd4u);
+  const __m128i struct_mask = _mm_set1_epi8(32);
+
+  __m128i white0 = _mm_cmpeq_epi8(in.v0, _mm_shuffle_epi8(white_table, in.v0));
+  __m128i white1 = _mm_cmpeq_epi8(in.v1, _mm_shuffle_epi8(white_table, in.v1));
+  __m128i white2 = _mm_cmpeq_epi8(in.v2, _mm_shuffle_epi8(white_table, in.v2));
+  __m128i white3 = _mm_cmpeq_epi8(in.v3, _mm_shuffle_epi8(white_table, in.v3));
+  uint64_t ws_res_0 = _mm_movemask_epi8(white0);
+  uint64_t ws_res_1 = _mm_movemask_epi8(white1);
+  uint64_t ws_res_2 = _mm_movemask_epi8(white2);
+  uint64_t ws_res_3 = _mm_movemask_epi8(white3);
+
+  whitespace =
+      (ws_res_0 | (ws_res_1 << 16) | (ws_res_2 << 32) | (ws_res_3 << 48));
+
+  __m128i struct1_r1 = _mm_add_epi8(struct_offset, in.v0);
+  __m128i struct2_r1 = _mm_add_epi8(struct_offset, in.v1);
+  __m128i struct3_r1 = _mm_add_epi8(struct_offset, in.v2);
+  __m128i struct4_r1 = _mm_add_epi8(struct_offset, in.v3);
+
+  __m128i struct1_r2 = _mm_or_si128(in.v0, struct_mask);
+  __m128i struct2_r2 = _mm_or_si128(in.v1, struct_mask);
+  __m128i struct3_r2 = _mm_or_si128(in.v2, struct_mask);
+  __m128i struct4_r2 = _mm_or_si128(in.v3, struct_mask);
+
+  __m128i struct1_r3 = _mm_shuffle_epi8(structural_table, struct1_r1);
+  __m128i struct2_r3 = _mm_shuffle_epi8(structural_table, struct2_r1);
+  __m128i struct3_r3 = _mm_shuffle_epi8(structural_table, struct3_r1);
+  __m128i struct4_r3 = _mm_shuffle_epi8(structural_table, struct4_r1);
+
+  __m128i struct1 = _mm_cmpeq_epi8(struct1_r2, struct1_r3);
+  __m128i struct2 = _mm_cmpeq_epi8(struct2_r2, struct2_r3);
+  __m128i struct3 = _mm_cmpeq_epi8(struct3_r2, struct3_r3);
+  __m128i struct4 = _mm_cmpeq_epi8(struct4_r2, struct4_r3);
+
+  uint64_t structural_res_0 = _mm_movemask_epi8(struct1);
+  uint64_t structural_res_1 = _mm_movemask_epi8(struct2);
+  uint64_t structural_res_2 = _mm_movemask_epi8(struct3);
+  uint64_t structural_res_3 = _mm_movemask_epi8(struct4);
+
+  structurals = (structural_res_0 | (structural_res_1 << 16) |
+                 (structural_res_2 << 32) | (structural_res_3 << 48));
+}
+
+} // namespace simdjson
+UNTARGET_REGION
+
+#endif // IS_X86_64
+#endif // SIMDJSON_STAGE1_FIND_MARKS_WESTMERE_H

include/simdjson/stage2_build_tape.h

@@ -1,16 +1,85 @@
 #ifndef SIMDJSON_STAGE2_BUILD_TAPE_H
 #define SIMDJSON_STAGE2_BUILD_TAPE_H
 
+#include <cassert>
+#include <cstring>
+#include <iostream>
+
 #include "simdjson/common_defs.h"
+#include "simdjson/jsoncharutils.h"
+#include "simdjson/numberparsing.h"
+#include "simdjson/parsedjson.h"
+#include "simdjson/simdjson.h"
+#include "simdjson/stringparsing.h"
 
-struct ParsedJson;
-
+namespace simdjson {
 void init_state_machine();
 
 WARN_UNUSED
-int unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj);
+really_inline bool is_valid_true_atom(const uint8_t *loc) {
+  uint64_t tv = *reinterpret_cast<const uint64_t *>("true    ");
+  uint64_t mask4 = 0x00000000ffffffff;
+  uint32_t error = 0;
+  uint64_t
+      locval; // we want to avoid unaligned 64-bit loads (undefined in C/C++)
+  // this can read up to 7 bytes beyond the buffer size, but we require
+  // SIMDJSON_PADDING of padding
+  static_assert(sizeof(uint64_t) - 1 <= SIMDJSON_PADDING);
+  std::memcpy(&locval, loc, sizeof(uint64_t));
+  error = (locval & mask4) ^ tv;
+  error |= is_not_structural_or_whitespace(loc[4]);
+  return error == 0;
+}
 
 WARN_UNUSED
-int unified_machine(const char *buf, size_t len, ParsedJson &pj);
+really_inline bool is_valid_false_atom(const uint8_t *loc) {
+  // We have to use an integer constant because the space in the cast
+  // below would lead to values illegally being qualified
+  // uint64_t fv = *reinterpret_cast<const uint64_t *>("false   ");
+  // using this constant (that is the same false) but nulls out the
+  // unused bits solves that
+  uint64_t fv = 0x00000065736c6166; // takes into account endianness
+  uint64_t mask5 = 0x000000ffffffffff;
+  // we can't use the 32 bit value for checking for errors otherwise
+  // the last character of false (it being 5 byte long!) would be
+  // ignored
+  uint64_t error = 0;
+  uint64_t
+      locval; // we want to avoid unaligned 64-bit loads (undefined in C/C++)
+  // this can read up to 7 bytes beyond the buffer size, but we require
+  // SIMDJSON_PADDING of padding
+  static_assert(sizeof(uint64_t) - 1 <= SIMDJSON_PADDING);
+  std::memcpy(&locval, loc, sizeof(uint64_t));
+  error = (locval & mask5) ^ fv;
+  error |= is_not_structural_or_whitespace(loc[5]);
+  return error == 0;
+}
+
+WARN_UNUSED
+really_inline bool is_valid_null_atom(const uint8_t *loc) {
+  uint64_t nv = *reinterpret_cast<const uint64_t *>("null    ");
+  uint64_t mask4 = 0x00000000ffffffff;
+  uint32_t error = 0;
+  uint64_t
+      locval; // we want to avoid unaligned 64-bit loads (undefined in C/C++)
+  // this can read up to 7 bytes beyond the buffer size, but we require
+  // SIMDJSON_PADDING of padding
+  static_assert(sizeof(uint64_t) - 1 <= SIMDJSON_PADDING);
+  std::memcpy(&locval, loc, sizeof(uint64_t));
+  error = (locval & mask4) ^ nv;
+  error |= is_not_structural_or_whitespace(loc[4]);
+  return error == 0;
+}
+
+template <Architecture T = Architecture::NATIVE>
+WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER int
+unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj);
+
+template <Architecture T = Architecture::NATIVE>
+int unified_machine(const char *buf, size_t len, ParsedJson &pj) {
+  return unified_machine<T>(reinterpret_cast<const uint8_t *>(buf), len, pj);
+}
+
+} // namespace simdjson
 
 #endif

include/simdjson/stringparsing.h

@@ -5,7 +5,13 @@
 #include "simdjson/jsoncharutils.h"
 #include "simdjson/parsedjson.h"
 
+#ifdef JSON_TEST_STRINGS
+void found_string(const uint8_t *buf, const uint8_t *parsed_begin,
+                  const uint8_t *parsed_end);
+void found_bad_string(const uint8_t *buf);
+#endif
 
+namespace simdjson {
 // begin copypasta
 // These chars yield themselves: " \ /
 // b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab
@@ -32,7 +38,6 @@ static const uint8_t escape_map[256] = {
     0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
 };
 
-
 // handle a unicode codepoint
 // write appropriate values into dest
 // src will advance 6 bytes or 12 bytes
@@ -40,9 +45,10 @@ static const uint8_t escape_map[256] = {
 // return true if the unicode codepoint was valid
 // We work in little-endian then swap at write time
 WARN_UNUSED
-really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, uint8_t **dst_ptr) {
+really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
+                                            uint8_t **dst_ptr) {
   // hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
-  // conversion isn't valid; we defer the check for this to inside the 
+  // conversion isn't valid; we defer the check for this to inside the
   // multilingual plane check
   uint32_t code_point = hex_to_u32_nocheck(*src_ptr + 2);
   *src_ptr += 6;
@@ -53,14 +59,14 @@ really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, uint8_t **d
       return false;
     }
     uint32_t code_point_2 = hex_to_u32_nocheck(*src_ptr + 2);
-    
+
     // if the first code point is invalid we will get here, as we will go past
     // the check for being outside the Basic Multilingual plane. If we don't
-    // find a \u immediately afterwards we fail out anyhow, but if we do, 
+    // find a \u immediately afterwards we fail out anyhow, but if we do,
     // this check catches both the case of the first code point being invalid
     // or the second code point being invalid.
     if ((code_point | code_point_2) >> 16) {
-        return false;
+      return false;
     }
 
     code_point =
@@ -72,97 +78,28 @@ really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, uint8_t **d
   return offset > 0;
 }
 
-WARN_UNUSED
-really_inline  bool parse_string(const uint8_t *buf, UNUSED size_t len,
-                                ParsedJson &pj, UNUSED const uint32_t depth, uint32_t offset) {
-#ifdef SIMDJSON_SKIPSTRINGPARSING // for performance analysis, it is sometimes useful to skip parsing
-  pj.write_tape(0, '"');// don't bother with the string parsing at all
-  return true; // always succeeds
-#else
-  pj.write_tape(pj.current_string_buf_loc - pj.string_buf, '"');
-  const uint8_t *src = &buf[offset + 1]; // we know that buf at offset is a "
-  uint8_t *dst = pj.current_string_buf_loc + sizeof(uint32_t);
-  const uint8_t *const start_of_string = dst;
-  while (1) {
-    __m256i v = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src));
-    // store to dest unconditionally - we can overwrite the bits we don't like
-    // later
-    _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), v);
-    auto bs_bits =
-        static_cast<uint32_t>(_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, _mm256_set1_epi8('\\'))));
-    auto quote_mask = _mm256_cmpeq_epi8(v, _mm256_set1_epi8('"'));
-    auto quote_bits =
-        static_cast<uint32_t>(_mm256_movemask_epi8(quote_mask));
-    if(((bs_bits - 1) & quote_bits) != 0 ) {
-      // we encountered quotes first. Move dst to point to quotes and exit
+// Holds backslashes and quotes locations.
+struct parse_string_helper {
+  uint32_t bs_bits;
+  uint32_t quote_bits;
+};
 
-      // find out where the quote is...
-      uint32_t quote_dist = trailingzeroes(quote_bits);
+// Finds where the backslashes and quotes are located.
+template <Architecture>
+parse_string_helper find_bs_bits_and_quote_bits(const uint8_t *src,
+                                                uint8_t *dst);
 
-      // NULL termination is still handy if you expect all your strings to be NULL terminated?
-      // It comes at a small cost
-      dst[quote_dist] = 0; 
+template <Architecture T>
+WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER
+    really_inline bool
+    parse_string(UNUSED const uint8_t *buf, UNUSED size_t len, ParsedJson &pj,
+                 UNUSED const uint32_t depth, UNUSED uint32_t offset);
 
-      uint32_t str_length = (dst - start_of_string) + quote_dist; 
-      memcpy(pj.current_string_buf_loc,&str_length, sizeof(uint32_t));
-      ///////////////////////
-      // Above, check for overflow in case someone has a crazy string (>=4GB?)
-      // But only add the overflow check when the document itself exceeds 4GB
-      // Currently unneeded because we refuse to parse docs larger or equal to 4GB.
-      ////////////////////////
-
-      
-      // we advance the point, accounting for the fact that we have a NULl termination
-      pj.current_string_buf_loc = dst + quote_dist + 1;
-
-#ifdef JSON_TEST_STRINGS // for unit testing
-      foundString(buf + offset,start_of_string,pj.current_string_buf_loc - 1);
-#endif // JSON_TEST_STRINGS
-      return true;
-    } 
-    if(((quote_bits - 1) & bs_bits ) != 0 ) {
-      // find out where the backspace is
-      uint32_t bs_dist = trailingzeroes(bs_bits);
-      uint8_t escape_char = src[bs_dist + 1];
-      // we encountered backslash first. Handle backslash
-      if (escape_char == 'u') {
-        // move src/dst up to the start; they will be further adjusted
-        // within the unicode codepoint handling code.
-        src += bs_dist;
-        dst += bs_dist;
-        if (!handle_unicode_codepoint(&src, &dst)) {
-#ifdef JSON_TEST_STRINGS // for unit testing
-          foundBadString(buf + offset);
-#endif // JSON_TEST_STRINGS
-          return false;
-        }
-      } else {
-        // simple 1:1 conversion. Will eat bs_dist+2 characters in input and
-        // write bs_dist+1 characters to output
-        // note this may reach beyond the part of the buffer we've actually
-        // seen. I think this is ok
-        uint8_t escape_result = escape_map[escape_char];
-        if (escape_result == 0u) {
-#ifdef JSON_TEST_STRINGS // for unit testing
-          foundBadString(buf + offset);
-#endif // JSON_TEST_STRINGS
-          return false; // bogus escape value is an error
-        }
-        dst[bs_dist] = escape_result;
-        src += bs_dist + 2;
-        dst += bs_dist + 1;
-      }
-    } else {
-      // they are the same. Since they can't co-occur, it means we encountered
-      // neither.
-      src += 32;
-      dst += 32;
-    }
-  }
-  // can't be reached
-  return true;
-#endif // SIMDJSON_SKIPSTRINGPARSING
-}
+} // namespace simdjson
 
+/// Now include the specializations:
+#include "simdjson/stringparsing_arm64.h"
+#include "simdjson/stringparsing_haswell.h"
+#include "simdjson/stringparsing_westmere.h"
 
 #endif

include/simdjson/stringparsing_arm64.h

@@ -0,0 +1,56 @@
+#ifndef SIMDJSON_STRINGPARSING_ARM64_H
+#define SIMDJSON_STRINGPARSING_ARM64_H
+
+#include "simdjson/stringparsing.h"
+#include "simdjson/stringparsing_macros.h"
+
+#ifdef IS_ARM64
+namespace simdjson {
+template <>
+really_inline parse_string_helper
+find_bs_bits_and_quote_bits<Architecture::ARM64>(const uint8_t *src,
+                                                 uint8_t *dst) {
+  // this can read up to 31 bytes beyond the buffer size, but we require
+  // SIMDJSON_PADDING of padding
+  static_assert(2 * sizeof(uint8x16_t) - 1 <= SIMDJSON_PADDING);
+  uint8x16_t v0 = vld1q_u8(src);
+  uint8x16_t v1 = vld1q_u8(src + 16);
+  vst1q_u8(dst, v0);
+  vst1q_u8(dst + 16, v1);
+
+  uint8x16_t bs_mask = vmovq_n_u8('\\');
+  uint8x16_t qt_mask = vmovq_n_u8('"');
+  const uint8x16_t bit_mask = {0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
+                               0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80};
+  uint8x16_t cmp_bs_0 = vceqq_u8(v0, bs_mask);
+  uint8x16_t cmp_bs_1 = vceqq_u8(v1, bs_mask);
+  uint8x16_t cmp_qt_0 = vceqq_u8(v0, qt_mask);
+  uint8x16_t cmp_qt_1 = vceqq_u8(v1, qt_mask);
+
+  cmp_bs_0 = vandq_u8(cmp_bs_0, bit_mask);
+  cmp_bs_1 = vandq_u8(cmp_bs_1, bit_mask);
+  cmp_qt_0 = vandq_u8(cmp_qt_0, bit_mask);
+  cmp_qt_1 = vandq_u8(cmp_qt_1, bit_mask);
+
+  uint8x16_t sum0 = vpaddq_u8(cmp_bs_0, cmp_bs_1);
+  uint8x16_t sum1 = vpaddq_u8(cmp_qt_0, cmp_qt_1);
+  sum0 = vpaddq_u8(sum0, sum1);
+  sum0 = vpaddq_u8(sum0, sum0);
+  return {
+      vgetq_lane_u32(vreinterpretq_u32_u8(sum0), 0), // bs_bits
+      vgetq_lane_u32(vreinterpretq_u32_u8(sum0), 1)  // quote_bits
+  };
+}
+
+template <>
+WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER
+    really_inline bool
+    parse_string<Architecture::ARM64>(UNUSED const uint8_t *buf,
+                                      UNUSED size_t len, ParsedJson &pj,
+                                      UNUSED const uint32_t depth,
+                                      UNUSED uint32_t offset) {
+  PARSE_STRING(Architecture::ARM64, buf, len, pj, depth, offset);
+}
+} // namespace simdjson
+#endif
+#endif

include/simdjson/stringparsing_haswell.h

@@ -0,0 +1,43 @@
+#ifndef SIMDJSON_STRINGPARSING_HASWELL_H
+#define SIMDJSON_STRINGPARSING_HASWELL_H
+
+#include "simdjson/stringparsing.h"
+#include "simdjson/stringparsing_macros.h"
+
+#ifdef IS_X86_64
+TARGET_HASWELL
+namespace simdjson {
+template <>
+really_inline parse_string_helper
+find_bs_bits_and_quote_bits<Architecture::HASWELL>(const uint8_t *src,
+                                                   uint8_t *dst) {
+  // this can read up to 31 bytes beyond the buffer size, but we require
+  // SIMDJSON_PADDING of padding
+  static_assert(sizeof(__m256i) - 1 <= SIMDJSON_PADDING);
+  __m256i v = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src));
+  // store to dest unconditionally - we can overwrite the bits we don't like
+  // later
+  _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), v);
+  auto quote_mask = _mm256_cmpeq_epi8(v, _mm256_set1_epi8('"'));
+  return {
+      static_cast<uint32_t>(_mm256_movemask_epi8(
+          _mm256_cmpeq_epi8(v, _mm256_set1_epi8('\\')))),     // bs_bits
+      static_cast<uint32_t>(_mm256_movemask_epi8(quote_mask)) // quote_bits
+  };
+}
+
+template <>
+WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER
+    really_inline bool
+    parse_string<Architecture::HASWELL>(UNUSED const uint8_t *buf,
+                                        UNUSED size_t len, ParsedJson &pj,
+                                        UNUSED const uint32_t depth,
+                                        UNUSED uint32_t offset) {
+  PARSE_STRING(Architecture::HASWELL, buf, len, pj, depth, offset);
+}
+
+} // namespace simdjson
+UNTARGET_REGION
+#endif
+
+#endif

include/simdjson/stringparsing_macros.h

@@ -0,0 +1,88 @@
+#ifndef SIMDJSON_STRINGPARSING_MACROS_H
+#define SIMDJSON_STRINGPARSING_MACROS_H
+
+// We need to compile that code for multiple architectures. However, target
+// attributes can be used only once by function definition. Huge macro seemed
+// better than huge code duplication.ç
+// bool PARSE_STRING(Architecture T, const uint8_t *buf, size_t len, ParsedJson
+//                  &pj,const uint32_t depth, uint32_t offset)
+#define PARSE_STRING(T, buf, len, pj, depth, offset)                           \
+  {                                                                            \
+    pj.write_tape(pj.current_string_buf_loc - pj.string_buf, '"');             \
+    const uint8_t *src =                                                       \
+        &buf[offset + 1]; /* we know that buf at offset is a " */              \
+    uint8_t *dst = pj.current_string_buf_loc + sizeof(uint32_t);               \
+    const uint8_t *const start_of_string = dst;                                \
+    while (1) {                                                                \
+      parse_string_helper helper = find_bs_bits_and_quote_bits<T>(src, dst);   \
+      if (((helper.bs_bits - 1) & helper.quote_bits) != 0) {                   \
+        /* we encountered quotes first. Move dst to point to quotes and exit   \
+         */                                                                    \
+                                                                               \
+        /* find out where the quote is... */                                   \
+        uint32_t quote_dist = trailing_zeroes(helper.quote_bits);              \
+                                                                               \
+        /* NULL termination is still handy if you expect all your strings to   \
+         * be NULL terminated? */                                              \
+        /* It comes at a small cost */                                         \
+        dst[quote_dist] = 0;                                                   \
+                                                                               \
+        uint32_t str_length = (dst - start_of_string) + quote_dist;            \
+        memcpy(pj.current_string_buf_loc, &str_length, sizeof(uint32_t));      \
+        /*****************************                                         \
+         * Above, check for overflow in case someone has a crazy string        \
+         * (>=4GB?)                 _                                          \
+         * But only add the overflow check when the document itself exceeds    \
+         * 4GB                                                                 \
+         * Currently unneeded because we refuse to parse docs larger or equal  \
+         * to 4GB.                                                             \
+         ****************************/                                         \
+                                                                               \
+        /* we advance the point, accounting for the fact that we have a NULL   \
+         * termination         */                                              \
+        pj.current_string_buf_loc = dst + quote_dist + 1;                      \
+        return true;                                                           \
+      }                                                                        \
+      if (((helper.quote_bits - 1) & helper.bs_bits) != 0) {                   \
+        /* find out where the backspace is */                                  \
+        uint32_t bs_dist = trailing_zeroes(helper.bs_bits);                    \
+        uint8_t escape_char = src[bs_dist + 1];                                \
+        /* we encountered backslash first. Handle backslash */                 \
+        if (escape_char == 'u') {                                              \
+          /* move src/dst up to the start; they will be further adjusted       \
+             within the unicode codepoint handling code. */                    \
+          src += bs_dist;                                                      \
+          dst += bs_dist;                                                      \
+          if (!handle_unicode_codepoint(&src, &dst)) {                         \
+            return false;                                                      \
+          }                                                                    \
+        } else {                                                               \
+          /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and \
+           * write bs_dist+1 characters to output                              \
+           * note this may reach beyond the part of the buffer we've actually  \
+           * seen. I think this is ok */                                       \
+          uint8_t escape_result = escape_map[escape_char];                     \
+          if (escape_result == 0u) {                                           \
+            return false; /* bogus escape value is an error */                 \
+          }                                                                    \
+          dst[bs_dist] = escape_result;                                        \
+          src += bs_dist + 2;                                                  \
+          dst += bs_dist + 1;                                                  \
+        }                                                                      \
+      } else {                                                                 \
+        /* they are the same. Since they can't co-occur, it means we           \
+         * encountered neither. */                                             \
+        if constexpr (T == Architecture::WESTMERE) {                           \
+          src += 16;                                                           \
+          dst += 16;                                                           \
+        } else {                                                               \
+          src += 32;                                                           \
+          dst += 32;                                                           \
+        }                                                                      \
+      }                                                                        \
+    }                                                                          \
+    /* can't be reached */                                                     \
+    return true;                                                               \
+  }
+
+#endif

include/simdjson/stringparsing_westmere.h

@@ -0,0 +1,41 @@
+#ifndef SIMDJSON_STRINGPARSING_WESTMERE_H
+#define SIMDJSON_STRINGPARSING_WESTMERE_H
+
+#include "simdjson/stringparsing.h"
+#include "simdjson/stringparsing_macros.h"
+
+#ifdef IS_X86_64
+TARGET_WESTMERE
+namespace simdjson {
+template <>
+really_inline parse_string_helper
+find_bs_bits_and_quote_bits<Architecture::WESTMERE>(const uint8_t *src,
+                                                    uint8_t *dst) {
+  // this can read up to 31 bytes beyond the buffer size, but we require
+  // SIMDJSON_PADDING of padding
+  __m128i v = _mm_loadu_si128(reinterpret_cast<const __m128i *>(src));
+  // store to dest unconditionally - we can overwrite the bits we don't like
+  // later
+  _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), v);
+  auto quote_mask = _mm_cmpeq_epi8(v, _mm_set1_epi8('"'));
+  return {
+      static_cast<uint32_t>(
+          _mm_movemask_epi8(_mm_cmpeq_epi8(v, _mm_set1_epi8('\\')))), // bs_bits
+      static_cast<uint32_t>(_mm_movemask_epi8(quote_mask)) // quote_bits
+  };
+}
+
+template <>
+WARN_UNUSED ALLOW_SAME_PAGE_BUFFER_OVERRUN_QUALIFIER LENIENT_MEM_SANITIZER
+    really_inline bool
+    parse_string<Architecture::WESTMERE>(UNUSED const uint8_t *buf,
+                                         UNUSED size_t len, ParsedJson &pj,
+                                         UNUSED const uint32_t depth,
+                                         UNUSED uint32_t offset) {
+  PARSE_STRING(Architecture::WESTMERE, buf, len, pj, depth, offset);
+}
+} // namespace simdjson
+UNTARGET_REGION
+#endif
+
+#endif

jsonchecker/adversarial/issue150/10.json

@@ -0,0 +1 @@
+[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[f

jsonchecker/adversarial/issue150/100.json

@@ -0,0 +1,5 @@
+{
+ "obj": { "a": 456e-000,
+    "a": "b",
+    "b": true,
+    "obj }[

jsonchecker/adversarial/issue150/1000.json

@@ -0,0 +1 @@
+{"\udB00\udF0CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC/CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC?CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCHCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC00et": "-0006106\u067b<06\u067b<6b\u0006106\u067b07b0,999ETd000000

jsonchecker/adversarial/issue150/1001.json

@@ -0,0 +1,42 @@
+{"web-app": {
+  "servlet": [   
+    {
+      "servlet-name": "cofaxCDS",
+      "servlet-class": "x.cds.CDSServlet",
+      "init-param": {
+        "installationAt": "Philadelphia, PA",
+        "adminEmail": "kscom",
+        "poweredBy": "Cofax",
+        "poweredByIcon": "/images/cofax.gif",
+        "saticPath": "/conten",
+        "temPKatePa///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////:////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////th": "templates",
+        "templ ateOverridePath": "",
+        "defaultListTemplate": false,
+        "defaultFileTemplate": true,
+        "useJSP": false,
+        ":spListTemplate": "listTemplate.jsp",
+        "jspFileTemplate": "articleTemplate.jsp",
+        "cachePackageTagsTrack": [1, 2, 3, 4],
+        "cachePagesTrack": 200,
+        "cachePagesStore": 100,
+        "cachePagesRefresh": 10,
+        "cachePagesDirtyRead": 10,
+        "searchEngineListTemplate": "forSearchEnginejList.htm",
+        "searchEngineFileTemplate": [{ "a": "b" }],
+        "searchEngineRobotsDb": "WEB-INF/robots.db",
+    "defaultFileTemplate": true,
+        "useJSP": false,
+        "jspListTemplate": "listTemplate.jsp",
+        "jspFileTemplate": "articleTemplate.jsp",
+        "cachePackageTagsTrack": [1, 2, 3, 4],
+        "cachePagesTrack": 200,
+        "eeeeeeegesStore": 100,
+        "ca@hePagesRefresh": 10,
+        "cachePagesDirtyRead": 10,
+        "searchEngineListTemplate": "forSearchEnginejList.htm",
+        "searchEngineFileTe/////////////////////////////////////////////////////////////////////////////////////////////////////////////th": "templates",
+        "templ ateOverridePath": "",
+        "defaultListTemplate": false,
+        "defaultFileTemplate": true,
+        "useJSP": false,
+        ":spLi99999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999977,-7779977,-77779999777,-7777999999999999999.99999999999999999999999999999999999999999999999999999999999999999997779999777,-7777999999999999999999999999999.999999999999999999999999999999999999999999999999999999999999977,-7779E77,-77779999777,-7777999999999999999999999999999.99999999999999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999977,-7779977,-77779999777,-7777999999999999999.99999999999999999999999999999999999999999999999999999999999999999997779999777,-7777999999999999999999999999999.99999999999999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999979999999999999.99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999499999999999999999999999888

jsonchecker/adversarial/issue150/1002.json

@@ -0,0 +1 @@
+  [  [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[{[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[t"

jsonchecker/adversarial/issue150/1003.json

@@ -0,0 +1,4 @@
+{"t": [   
+
+ -9923E00, 
+{"tE0Fp00

jsonchecker/adversarial/issue150/1004.json

@@ -0,0 +1,2 @@
+{"t": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[   
+ 99999.9990

jsonchecker/adversarial/issue150/1005.json

@@ -0,0 +1 @@
+{"\u00D0000\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\63CD

jsonchecker/adversarial/issue150/1006.json

@@ -0,0 +1 @@
+"\u0000003W333D

jsonchecker/adversarial/issue150/1007.json

@@ -0,0 +1,5 @@
+[[[{
+    "cofaxCDS": " ",
+    "cofaxEmail": "/cofaxutil/aemail/*",
+    "cofaxAdmin": "/admin/*",
+    "fileServlet": "/sta

jsonchecker/adversarial/issue150/1009.json

@@ -0,0 +1 @@
+[["

jsonchecker/adversarial/issue150/101.json

@@ -0,0 +1 @@
+"\r:zt\udadd\uddaddaud

jsonchecker/adversarial/issue150/1010.json

@@ -0,0 +1 @@
+{"C000\udB00\uDc08?000mud%0C000\udB00\uDc00l300\udB00\udF0C000\udB00\uDc000\udB00\uDc00l300udF0C000\udB00\uDc00l3000\udB00\uDc08?11111111111111111111111111111111111dB00\uDc08?11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111B00\uDc00l300udF0C000\udB00\uDc00l3000\udB00\uDc08?11111111111111111111111111111111111dB00\uDc08?11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111jsmListTemplate": "l\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\V\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\istTemp\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\>\\\\\\\\\\\\\\\\0C000\udB00\uDc000\udB00\uDc00l3\\\\\\\\\\\\0\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\^\\\\\\\\\\\\\\\\\\\\\\(

jsonchecker/adversarial/issue150/1011.json

@@ -0,0 +1 @@
+"\u00c00006D

jsonchecker/adversarial/issue150/1012.json

@@ -0,0 +1 @@
+[ 99429E10,99929999999999949999999999999999999999E109,9999E10,"}}`

jsonchecker/adversarial/issue150/1015.json

@@ -0,0 +1 @@
+"\b<.E\u00D0=6\"\b<.E\u099999999.9999999999999999999999999999999999999999999999999997779999777,-77779999999999999999999.99999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999977,-7779977,-77779999777,-7777999999999999999.99999999999999999999999999999999999999999999999999999999999999999997779999777,-77779999999999999999999.99999999999999999999999999999999999999999999>999999999999999977,-7779977,-77779999777,-79999999999999999999999.99999999999999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999977,-7779977,-77779999777,-7777999999999999999.99999999999999999999999999999999999999999999997779999777,-77779999999999999999999.9999999999999999999999999999999999999#999999999999977,-777799999999,77959990 ,9999977,-7779977,-77779999777,-7777999999999999999.99999999999999999999999999999999999999999999999999999999999999999997779999777,-77779999999999999999999.999999999999999999999999999999999999999999999999999999999999977,-7779977,-77779999777,-79999999999999999999999.99999999999999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999977,-7779977,-77779999777,-7777999999999999999.99999999999999999999999999999999999999999999999999999999999999999997779999777,-7777999999999999999999999999999.99999999999999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999979999999999999.999999997,-7777999999999999999.99999999999999999999099999999999999999999999999999999999999999999997779999777,-7777999999999999999999999999999.99999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999977,-7779977,-77779999777,-7777999999999999999.99999999999999999999999999999999999999999999999999999999999999999997779999777,-7777999999999999999999999999999.9999999999999999999999999999999999999999999999999999999999999799999999997,-7779977,-77779999777,-7777999999999999999999999999999.99999999999999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999977,-7779977,-77779999777,-7777779999777,-7777999999999999999999999999999.999y9999999999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999979999999999999.999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999777,-7777999wwwwwwwwwwwwwwwwwwwwwwwwwwwwww999999999999999999999999.99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999997779999777,-7777999999999999999999999999999.99999999999999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999979999999999999.999999997,-7777999999999999999.99999999999999999999099999999999999999999999999999999999999999999997779999777,-7777999999999999999999999999999.99999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999977999999999999999999.99999999999999,-7779977,-77779999777,-7777999999999999999.99999999999999999999999999999999999999999999999999999999999999999997779999777,-7777999999999999999999999999999.999999999999999999999999999999999999999999999999999999999999977,-7779977,-77779999777,-7777999999999999999999999999999.99999999999999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999977,-7779977,-77779999777,-7777999999999999999.99999999999999999999999999999999999999999999999999999999999999999997779999777,-7777999999999999999999999999999.99999999999999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999979999999999999.999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999777,-7777999999999999999999999999999.99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999998799999999,77959990 ,999988

jsonchecker/adversarial/issue150/1016.json

@@ -0,0 +1 @@
+[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[nR

jsonchecker/adversarial/issue150/1017.json

@@ -0,0 +1,5 @@
+{"wrvle-app": {
+  "sevle6app": {
+  "servlet": [   
+    {
+      "servlet0h006000006\u067b<MOu06At": ["}}}

jsonchecker/adversarial/issue150/1018.json

@@ -0,0 +1,2 @@
+[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[C g,
+[}}

jsonchecker/adversarial/issue150/1019.json

@@ -0,0 +1 @@
+{"\udB00\udF0000}

jsonchecker/adversarial/issue150/102.json

@@ -0,0 +1 @@
+[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[-

jsonchecker/adversarial/issue150/1020.json

@@ -0,0 +1,3 @@
+{"wzb-app": {
+  "et-name": "cofaxCDS",
+      "sXrvlet-\udc00000

jsonchecker/adversarial/issue150/1021.json

@@ -0,0 +1 @@
+{"\udB00\udF0C00et": "/t-}}}

jsonchecker/adversarial/issue150/1022.json

@@ -0,0 +1 @@
+[ 99429E1,9910,69911111119929E+0,"}=}

jsonchecker/adversarial/issue150/1023.json

@@ -0,0 +1,3 @@
+{"weeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeegaglib": {
+    "taglib-uri": "cofax.tld",
+    "taglib-lo

jsonchecker/adversarial/issue150/1024.json

@@ -0,0 +1 @@
+[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[9,99999999399999.999999.9,3

jsonchecker/adversarial/issue150/1025.json

@@ -0,0 +1 @@
+[false,false,"}}}

jsonchecker/adversarial/issue150/1026.json

@@ -0,0 +1 @@
+["\udB00\udF0C000\udB00\udc080$\udc08?0$000c08?0$\udc08?00CN0060000(00400000006000067b<MO/\u000000CN006000006\u0000dc000600006\u0000dc08?0$\udc08?0$0000000000\udB00\udc08?0$000000dc08?0$\udc0003

jsonchecker/adversarial/issue150/1027.json

@@ -0,0 +1 @@
+["\ud6000\udc08?0$800000dc08?0$\udc000dc08?0$\udc083

jsonchecker/adversarial/issue150/1028.json

@@ -0,0 +1 @@
+[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[	

jsonchecker/adversarial/issue150/1029.json

@@ -0,0 +1,23 @@
+{"web-app": {
+  "servlet": [   
+    {
+      "servlet-name": "cofaxCDS",
+      "servlet-class": "x.cds.CDSServlet",
+      "init-param": {
+        "installationAt": "Philadelphia, PA",
+        "adminEmail": "kscom",
+        "poweredBy": "Cofax",
+        "poweredByIcon": "/images/cofax.gif",
+        "staticPath": "/conten",
+        "templatePath": "templates",
+        "templateOverridePath": "",
+        "defaultListTemplate": false,
+        "yefaultFileTemplate": true,
+        "useJSP": false,
+        "jspListTemplate": "listTemplate.jsp",
+        "jspFileTemplate": "articleTemplate.jsp",
+        "cachePackageTagsTrack": [1, 2, 3, 4],
+        "cachePagesTrack": 200,
+        "cachePagesStore": 100,
+        "cachePagesRefresh": 10,
+        "cachePage9999999999977,-777799999999,77959990 ,9999977,-7779977,-77779999777,-7777999999999999999.39999999999999999999999999999999999999999999999999999999999999999997779999777,-7777999999999999999999999999999.99999999999999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999979999999999999.9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999977,-7779977,-77779999777,-7777999999999999999.999999999999999999999999999899999999999999977E9999777,-7777999999999999999999Y99999999.99999999999999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999979999999999999.9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999977,-7779977,-77779999777,-7777999999999999999.99999999999999999999999999999999999999999999999999999999999999999997779999777,-7777999999999999999999999999999.999999999999999999999999999999999999999999999999999999999999977,-777KE77,-77779999777,-7777999999999999999999999999999.99999999999999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999977,-7779977,-77779999777,-7777999999999999999.99999999999999999999999999999999999999999999999999999999999999999997779999777,-7777999999999999999999999999999.99999999999999999999999%99999999999999999999999999999999999977,-777799999999,77959990 ,9999979999999999999.999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999

jsonchecker/adversarial/issue150/103.json

@@ -0,0 +1,2 @@
+{
+    "a                          

jsonchecker/adversarial/issue150/1030.json

@@ -0,0 +1 @@
+[ 9992777777,-777799999999,77959990 ,9999977,"7779977,-77779999777,-7777999999999999999999999999999.99999999999999999999999999999999999999999999999999999999999977,-777799999999,77959990 ,9999977,-7779977,-77779999777,-7777999999999999999.999999999999999999999999999999999999999999999992777777,-777799999999,77959990 ,9999977,-7779977,-77779999777,-7777999999999999999999999999999.99999999999999999999999999999999999999999A99999999999999999977,-777799999999,77959990 ,9999977,-7779977,-77779999777,-7777999999999999999.99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999V99999999888

jsonchecker/adversarial/issue150/1032.json

@@ -0,0 +1,5 @@
+[[[[[[[[[[ ],
+[[[[[[[[[[[ ],
+10,[[[[ ],
+[[[[[[[[[[[ ],
+10,"9H0T0000A-}

jsonchecker/adversarial/issue150/1033.json

@@ -0,0 +1 @@
+[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[0

jsonchecker/adversarial/issue150/1035.json

@@ -0,0 +1 @@
+"\uB0000006Ab#U>d WE

jsonchecker/adversarial/issue150/1036.json

@@ -0,0 +1,79 @@
+{"web-app": {
+  "s\rvlet": [   
+    {
+      "servlet-name": "cofaxCDS",
+      "servlet-class": "x.cds.CDSServlet",
+      "init-param": {
+        "installationAt": "Philadelphia, PA",
+        "adminEmail": "kscom",
+        "poweredBy": "Cofax",
+        "poweredByIcon": "/images/cofax.gif",
+        "staticPath": "/conten",
+        "templatePath": "templates",
+        "templateOverridePath": "",
+        "defaultListTemplate": false,
+        "defaultFileTemplate": true,
+        "useJSP": false,
+        "jspListTemplate": "listTemplate.jsp",
+        "jspFileTemplate": "articleTemplate.jsp",
+        "cachete": "forSearchEnginesList.htm",
+        "searchEngineFileTemplate": [{ "a": "b" }],
+        "searchEngineRobotsDb": "WEB-INF/robots.db",
+        "useDataStore": true,
+        "dataStoreClass": "org.cofax.SqlDataStore",
+        "redirectionClass": "org.cofax.SqlRedirection",
+        "dataStoreName": "cofax",
+        "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver",
+        "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon",
+        "dataStoreUser": "sa",
+        "dataStorePassword": "dataStoreTestQuery",
+        "dataStoreTestQuery": "SET NOCOUNT ON;select tofaxToolest='test';",
+        "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log",
+        "dataStoreInitConns": 10,
+        "dataStoreMaxConns": 100,
+        "dataStoreConnUsageLimit": 100,
+        "dataStoreLogLevel": "debug",
+        "maxUrlLength": 500}},
+    {
+      "servlet-name": "cofaxEmail",
+      "servlet-class": "org.cofax.cds.EmailServlet",
+      "init-param": {
+      "mailHost": "mail1",
+      "mailHostOverride": "mail2"}},
+    {
+      "servlet-name": "cofaxAdmin",
+      "servlet-class": "org.cofax.cds.AdminServlet",
+      "option": true},
+ 
+    {
+      "servlet-name": "fileServlet",
+      "option": false,
+      "servs": "org.cofax.cds.FileServlet"},
+    {
+      "servlet-name": "cofaxTools",
+      "servl": "org.cofax.cms.CofaxToolsServlet",
+      "asdasd": 123123,
+      "init-param": {
+        "temh": "tootes/",
+        "log": 1,
+        "logLocation": "/usr/local/tomcat/logs/CofaxTools.log",
+        "logMaxSize": "",
+        "cPath": "/contdataLog",
+        "logLocation": "/uusr/local/tomcat/logs/dataLog.log",
+        "dataLe": "",
+        "removePageCache": "/content/admin/remove?cache=pages&id=",
+        "removeTemplateCache": "/content/admin/remove?cache=templates&id=",
+        "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder",
+        "lookInContext": 1,
+        "adminGroupID": 4,
+        "betaServer": true}}],
+  "servlet-mapping": {
+    "cofaxCDS": "/",
+    "cofaxEmail": "/cofaxutil/aemail/*",
+    "cofaxAdmin": "/admin/*",
+    "fileServlet": "/static/*",
+    "cofaxTools": "/tools/*"},
+ 
+  "taglib": {
+    "taglib-uri": "cofax.tld",
+    "ta}lib-location": "

jsonchecker/adversarial/issue150/1037.json

@@ -0,0 +1 @@
+[ 99923E00,9999.999923E00,"}}}

jsonchecker/adversarial/issue150/1038.json

@@ -0,0 +1 @@
+[[[[[929E10,9,"/sta

jsonchecker/adversarial/issue150/1039.json

@@ -0,0 +1,59 @@
+{"web-app": {
+  "servlet": [   
+    {
+      "servlet-name": "cofaxCDS",
+      "servlet-class": "x.cds.CDSServlet",
+      "init-param": {
+        "installationAt": "Philadelphia, PA",
+        "adminEmail": "kscom",
+        "poweredBy": "Cofax",
+        "poweredByIcon": "/images/cofax.gif",
+        "staticPath": "/conten",
+        "templatePath": "templates",
+        "templateOv:microsoft:sqlservererridePath": "",
+        "defaultListTemplate": false,
+        "defaultFileTemplate": true,
+        "useJSP": false,
+        "jspListTemplate": "listTemplate.jsp",
+        "jspFileTemplate": "articleTemplate.jsp",
+        "cachePackageTagsTrack": [1, 2, 3, 4],
+        "cachePagesTrack": 200,
+        "cachePagesStore": 100,
+        "cachePagesRefresh": 10,
+        "cachePagesDirtyRead": 10,
+        "searchEnyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyygineListTemplate": "forSearchEnginesList.htm",
+        "search+ngineFileTemplate": [{ "a": "b" }],
+        "searchEngineRobotsDb": "WEB-INF/robots.db",
+        "useDataStore": true,
+        "dataStoreClass": "org.cofax.SqlDataStore",
+        "redirectionClass": "org.cofax.SqlRedirection",
+        "dataStoreName": "cofax",
+        "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver",
+        "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon",
+        "dataStoreUser": "sa",
+        "dataStorePassword": "dataStoreTestQuery",
+        "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';",
+        "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log",
+        "dataStoreInitConns": 10,
+        "dataStoreMaxConns": 100,
+        "dataStoreConnUsageLimit": 100,
+        "dataStoreLogLevel": "debug",
+        "maxUrlLength": 500}},
+    {
+      "servlet-name": "cofaxEmail",
+      "servlet-class": "org.cofax.cds.EmailServlet",
+      "init-param": {
+      "mailHost": "dail1",
+      "mailHostOverride": "mail2"}},
+    {
+      "servlet-name": "cofaxAdmin",
+      "servlet-class": "org.cofax.cds.AdminServlet",
+      "option": true},
+ 
+    {
+      "servlet-name": "fileServlet",
+      "option": false,
+      "servs": "org.cofax.cds.FileServlet"},
+    {
+      "min": "/admin/*",
+    "fileServlet": "/sta

jsonchecker/adversarial/issue150/104.json

@@ -0,0 +1,5 @@
+{
+":#t\udadd\udCdb": true,
+      "undef": null,
+    "c": false,
+ "