new release.

Removing garbage from script.
Store the string lengths on the string tape (#101 )
2026-06-08 17:27:07 +00:00 · 2019-03-13 20:02:44 -04:00 · 2019-03-13 20:02:44 -04:00 · 2019-03-13 19:32:57 -04:00 · 2019-03-13 11:01:31 -04:00 · 2019-03-13 08:15:27 -04:00
40 changed files with 1458 additions and 1255 deletions
@@ -21,7 +21,7 @@ jobs:

      - run:
          name: Running tests (gcc)
-          command: make quiettest
+          command: make quiettest amalgamate

      - run:
          name: Building (gcc, cmake)
@@ -58,7 +58,7 @@ jobs:

      - run:
          name: Running tests (clang)
-          command: make quiettest
+          command: make quiettest amalgamate

      - run:
          name: Building (clang, cmake)
@@ -7,3 +7,4 @@ steps:
  commands:
  - make -j2
  - make quiettest -j2
+  - make amalgamate
@@ -1,6 +1,6 @@
 # Set the default behavior, in case people don't have core.autocrlf set.
 # Uncomment next line to adjust line endings
-#* text=auto eol=lf
+* text=auto 

 # Explicitly declare text files you want to always be normalized and converted
 # to native line endings on checkout.
@@ -9,6 +9,11 @@ endif()

 project(simdjson)
 set(SIMDJSON_LIB_NAME simdjson)
+set(PROJECT_VERSION_MAJOR 0)
+set(PROJECT_VERSION_MINOR 1)
+set(PROJECT_VERSION_PATCH 0)
+set(SIMDJSON_LIB_VERSION "0.1.0" CACHE STRING "simdjson library version")
+set(SIMDJSON_LIB_SOVERSION "0" CACHE STRING "simdjson library soversion")

 if(NOT MSVC)
 option(SIMDJSON_BUILD_STATIC "Build a static library" OFF) # turning it on disables the production of a dynamic library
@@ -28,7 +28,7 @@ COMPARISONEXECUTABLES=minifiercompetition parsingcompetition parseandstatcompeti
 SUPPLEMENTARYEXECUTABLES=parse_noutf8validation parse_nonumberparsing parse_nostringparsing

 HEADERS= include/simdjson/simdutf8check.h include/simdjson/stringparsing.h include/simdjson/numberparsing.h include/simdjson/jsonparser.h include/simdjson/common_defs.h include/simdjson/jsonioutil.h benchmark/benchmark.h benchmark/linux/linux-perf-events.h include/simdjson/parsedjson.h include/simdjson/stage1_find_marks.h include/simdjson/stage2_build_tape.h include/simdjson/jsoncharutils.h include/simdjson/jsonformatutils.h
-LIBFILES=src/jsonioutil.cpp src/jsonparser.cpp src/stage1_find_marks.cpp         src/stage2_build_tape.cpp src/parsedjson.cpp src/parsedjsoniterator.cpp
+LIBFILES=src/jsonioutil.cpp src/jsonparser.cpp src/simdjson.cpp src/stage1_find_marks.cpp        src/stage2_build_tape.cpp src/parsedjson.cpp src/parsedjsoniterator.cpp
 MINIFIERHEADERS=include/simdjson/jsonminifier.h include/simdjson/simdprune_tables.h
 MINIFIERLIBFILES=src/jsonminifier.cpp

@@ -116,11 +116,11 @@ jsoncheck:tests/jsoncheck.cpp $(HEADERS) $(LIBFILES)
 	$(CXX) $(CXXFLAGS) -o jsoncheck $(LIBFILES) tests/jsoncheck.cpp -I. $(LIBFLAGS)

 numberparsingcheck:tests/numberparsingcheck.cpp $(HEADERS) $(LIBFILES)
-	$(CXX) $(CXXFLAGS) -o numberparsingcheck tests/numberparsingcheck.cpp  src/jsonioutil.cpp src/jsonparser.cpp src/stage1_find_marks.cpp  src/parsedjson.cpp       -I. $(LIBFLAGS) -DJSON_TEST_NUMBERS
+	$(CXX) $(CXXFLAGS) -o numberparsingcheck tests/numberparsingcheck.cpp  src/jsonioutil.cpp src/jsonparser.cpp src/simdjson.cpp src/stage1_find_marks.cpp  src/parsedjson.cpp       -I. $(LIBFLAGS) -DJSON_TEST_NUMBERS


 stringparsingcheck:tests/stringparsingcheck.cpp $(HEADERS) $(LIBFILES)
-	$(CXX) $(CXXFLAGS) -o stringparsingcheck tests/stringparsingcheck.cpp  src/jsonioutil.cpp src/jsonparser.cpp src/stage1_find_marks.cpp  src/parsedjson.cpp      -I. $(LIBFLAGS) -DJSON_TEST_STRINGS
+	$(CXX) $(CXXFLAGS) -o stringparsingcheck tests/stringparsingcheck.cpp  src/jsonioutil.cpp src/jsonparser.cpp src/simdjson.cpp src/stage1_find_marks.cpp  src/parsedjson.cpp      -I. $(LIBFLAGS) -DJSON_TEST_STRINGS


 minifiercompetition: benchmark/minifiercompetition.cpp $(HEADERS) $(LIBS) $(MINIFIERHEADERS) $(LIBFILES) $(MINIFIERLIBFILES)
@@ -138,13 +138,13 @@ jsonstats: tools/jsonstats.cpp $(HEADERS) $(LIBFILES)
 ujdecode.o: $(UJSON4C_INCLUDE)
 	$(CC) $(CFLAGS) -c dependencies/ujson4c/src/ujdecode.c

-parseandstatcompetition: benchmark/parseandstatcompetition.cpp $(HEADERS) $(LIBFILES)
+parseandstatcompetition: benchmark/parseandstatcompetition.cpp $(HEADERS) $(LIBFILES) $(LIBS)
 	$(CXX) $(CXXFLAGS)  -o parseandstatcompetition $(LIBFILES) benchmark/parseandstatcompetition.cpp -I. $(LIBFLAGS) $(COREDEPSINCLUDE)

-distinctuseridcompetition: benchmark/distinctuseridcompetition.cpp $(HEADERS) $(LIBFILES)
+distinctuseridcompetition: benchmark/distinctuseridcompetition.cpp $(HEADERS) $(LIBFILES) $(LIBS)
 	$(CXX) $(CXXFLAGS)  -o distinctuseridcompetition $(LIBFILES) benchmark/distinctuseridcompetition.cpp  -I. $(LIBFLAGS) $(COREDEPSINCLUDE)

-parsingcompetition: benchmark/parsingcompetition.cpp $(HEADERS) $(LIBFILES)
+parsingcompetition: benchmark/parsingcompetition.cpp $(HEADERS) $(LIBFILES) $(LIBS)
 	$(CXX) $(CXXFLAGS)  -o parsingcompetition $(LIBFILES) benchmark/parsingcompetition.cpp -I. $(LIBFLAGS) $(COREDEPSINCLUDE)

 allparsingcompetition: benchmark/parsingcompetition.cpp $(HEADERS) $(LIBFILES) $(EXTRAOBJECTS) $(LIBS)
@@ -1,52 +1,55 @@
-# simdjson : Parsing gigabytes of JSON per second
+#  simdjson : Parsing gigabytes of JSON per second
 [![Build Status](https://cloud.drone.io/api/badges/lemire/simdjson/status.svg)](https://cloud.drone.io/lemire/simdjson/)
-[![CircleCI](https://circleci.com/gh/lemire/simdjson.svg?style=svg)](https://circleci.com/gh/lemire/simdjson) 
+[![CircleCI](https://circleci.com/gh/lemire/simdjson.svg?style=svg)](https://circleci.com/gh/lemire/simdjson)
 [![Build Status](https://img.shields.io/appveyor/ci/lemire/simdjson.svg)](https://ci.appveyor.com/project/lemire/simdjson)
 [![][license img]][license]


-## A C++  library to see how fast we can parse JSON with complete validation.
+## A C++ library to see how fast we can parse JSON with complete validation.

 JSON documents are everywhere on the Internet. Servers spend a lot of time parsing these documents. We want to accelerate the parsing of JSON per se using commonly available SIMD instructions as much as possible while doing full validation (including character encoding).

+<img src="images/logo.png" width="10%">
+
+
 ## Paper

 A description of the design and implementation of simdjson appears at https://arxiv.org/abs/1902.08318 and an informal blog post providing some background and context is at https://branchfree.org/2019/02/25/paper-parsing-gigabytes-of-json-per-second/.
-## Some performance results

-We can use a quarter or fewer instructions than a state-of-the-art parser like RapidJSON, and half as many as sajson. To our knowledge, simdjson is the first fully-validating JSON parser to run at gigabytes per second on commodity processors.
+## Performance results
+
+simdjson uses three-quarters less instructions than state-of-the-art parser RapidJSON and fifty percent less than sajson. To our knowledge, simdjson is the first fully-validating JSON parser to run at gigabytes per second on commodity processors.

 <img src="doc/gbps.png" width="90%">

-
 On a Skylake processor, the parsing speeds (in GB/s) of various processors on the twitter.json file are as follows.

-| parser   |  GB/s |
-|---|---|
-| simdjson | 2.2	|
-| RapidJSON encoding-validation | 0.51|
-| RapidJSON encoding-validation, insitu | 0.71|
-| sajson (insitu, dynamic) | 0.70|
-| sajson (insitu, static) | 0.97|
-| dropbox | 0.14|
-| fastjson | 0.26|
-| gason | 0.85|
-| ultrajson | 0.42|
-| jsmn | 0.28|
-|cJSON | 0.34|
+| parser                                | GB/s |
+| ------------------------------------- | ---- |
+| simdjson                              | 2.2  |
+| RapidJSON encoding-validation         | 0.51 |
+| RapidJSON encoding-validation, insitu | 0.71 |
+| sajson (insitu, dynamic)              | 0.70 |
+| sajson (insitu, static)               | 0.97 |
+| dropbox                               | 0.14 |
+| fastjson                              | 0.26 |
+| gason                                 | 0.85 |
+| ultrajson                             | 0.42 |
+| jsmn                                  | 0.28 |
+| cJSON                                 | 0.34 |

 ## Requirements

 - We support platforms like Linux or macOS, as well as Windows through Visual Studio 2017 or later.
- A processor with AVX2 (i.e., Intel processors starting with the Haswell microarchitecture released 2013, and processors from AMD starting with the Ryzen)
+- A processor with AVX2 (i.e., Intel processors starting with the Haswell microarchitecture released 2013 and AMD processors starting with the Zen microarchitecture released 2017).
 - A recent C++ compiler (e.g., GNU GCC or LLVM CLANG or Visual Studio 2017), we assume C++17. GNU GCC 7 or better or LLVM's clang 6 or better.
- Some benchmark scripts assume bash and other common utilities, but they are  optional.
+- Some benchmark scripts assume bash and other common utilities, but they are optional.

 ## License

-This code is made available under the Apache License 2.0. 
+This code is made available under the Apache License 2.0.

-Under Windows, we build some tools using the  windows/dirent_portable.h  file (which is outside our library code): it under the liberal (business-friendly) MIT license.
+Under Windows, we build some tools using the windows/dirent_portable.h file (which is outside our library code): it under the liberal (business-friendly) MIT license.

 ## Code example

@@ -61,8 +64,12 @@ const char * filename = ... //
 std::string_view p = get_corpus(filename);
 ParsedJson pj;
 pj.allocateCapacity(p.size()); // allocate memory for parsing up to p.size() bytes
-bool is_ok = json_parse(p, pj); // do the parsing, return false on error
+const int res = json_parse(p, pj); // do the parsing, return 0 on success
 // parsing is done!
+if (res != 0) {
+    // You can use the "simdjson/simdjson.h" header to access the error message
+    std::cout << "Error parsing:" << simdjson::errorMsg(res) << std::endl;
+}
 // You can safely delete the string content
 free((void*)p.data());
 // the ParsedJson document can be used here
@@ -97,7 +104,7 @@ copy the files in your project in your include path. You can then include them q
 #include "simdjson.h"
 #include "simdjson.cpp"
 int main(int argc, char *argv[]) {
-  const char * filename = argv[1]; 
+  const char * filename = argv[1];
  std::string_view p = get_corpus(filename);
  ParsedJson pj = build_parsed_json(p); // do the parsing
  if( ! pj.isValid() ) {
@@ -109,8 +116,18 @@ int main(int argc, char *argv[]) {
 }
 ```

+We require hardware support for AVX2 instructions. You have to make sure that you instruct your 
+compiler to use these instructions as needed. Under compilers such as GNU GCC or LLVM clang, the
+flag `-march=native` used on a recent Intel processor (Haswell or better) is sufficient. For portability
+of the binary files you can also specify directly the Haswell processor (`-march=haswell`). You may 
+also use the flags `-mavx2 -mbmi2`. Under Visual Studio, you need to target x64 and add the 
+flag `/arch:AVX2`. 
+
+
 Note: In some settings, it might be desirable to precompile `simdjson.cpp` instead of including it.

+
+
 ## Usage (old-school Makefile on platforms like Linux or macOS)

 Requirements: recent clang or gcc, and make. We recommend at least GNU GCC/G++ 7 or LLVM clang 6. A system like Linux or macOS is expected.
@@ -122,12 +139,13 @@ make
 make test
 ```

-
 To run benchmarks:
+
 ```
 make parse
 ./parse jsonexamples/twitter.json
 ```
+
 Under Linux, the `parse` command gives a detailed analysis of the performance counters.

 To run comparative benchmarks (with other parsers):
@@ -138,7 +156,7 @@ make benchmark

 ## Usage (CMake on platforms like Linux or macOS)

-Requirements:  We require a recent version of cmake. On macOS, the easiest way to install cmake might be to use [brew](https://brew.sh) and then type 
+Requirements: We require a recent version of cmake. On macOS, the easiest way to install cmake might be to use [brew](https://brew.sh) and then type

 ```
 brew install cmake
@@ -146,15 +164,13 @@ brew install cmake

 There is an [equivalent brew on Linux which works the same way as well](https://linuxbrew.sh).

-You need a recent compiler like clang or gcc. We recommend at least GNU GCC/G++ 7 or LLVM clang 6. For example, you can install a recent compiler with brew: 
+You need a recent compiler like clang or gcc. We recommend at least GNU GCC/G++ 7 or LLVM clang 6. For example, you can install a recent compiler with brew:

 ```
 brew install gcc@8
 ```

-Optional: You  need to tell cmake which compiler you wish to use by setting the CC and CXX variables. Under bash, you can do so with commands such as ``export CC=gcc-7`` and ``export CXX=g++-7``. 
-
-
+Optional: You need to tell cmake which compiler you wish to use by setting the CC and CXX variables. Under bash, you can do so with commands such as `export CC=gcc-7` and `export CXX=g++-7`.

 Building: While in the project repository, do the following:

@@ -174,11 +190,10 @@ You can build a static library:
 mkdir buildstatic
 cd buildstatic
 cmake -DSIMDJSON_BUILD_STATIC=ON ..
-make 
+make
 make test
 ```

-
 In some cases, you may want to specify your compiler, especially if the default compiler on your system is too old. You may proceed as follows:

 ```
@@ -191,18 +206,47 @@ make
 make test
 ```

-
 ## Usage (CMake on Windows using Visual Studio)

-
-We are assuming that you have a common Windows PC with at least Visual Studio 2017, and an x64 processor with AVX2 support (2013 Haswell or later).
+We assume you have a common Windows PC with at least Visual Studio 2017 and an x64 processor with AVX2 support (2013 Intel Haswell or later).

 - Grab the simdjson code from GitHub, e.g., by cloning it using [GitHub Desktop](https://desktop.github.com/).
- Install [CMake](https://cmake.org/download/). When you install it, make sure to ask that ``cmake`` be made available from the command line. Please choose a recent version of cmake.
- Create a subdirectory within simdjson, such as ``VisualStudio``.
- Using a shell, go to this newly created directory. 
- Type ``cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..`` in the shell while in the ``VisualStudio`` repository. (Alternatively, if you want to build a DLL, you may use the command line ``cmake -DCMAKE_GENERATOR_PLATFORM=x64 -DSIMDJSON_BUILD_STATIC=OFF  ..``.)
- This last command created a Visual Studio solution file in the newly created directory (e.g., ``simdjson.sln``). Open this file in Visual Studio. You should now be able to build the project and run the tests. For example, in the ``Solution Explorer`` window (available from the ``View`` menu), right-click ``ALL_BUILD`` and select ``Build``. To test the code, still in the ``Solution Explorer`` window, select ``RUN_TESTS`` and select ``Build``.
+- Install [CMake](https://cmake.org/download/). When you install it, make sure to ask that `cmake` be made available from the command line. Please choose a recent version of cmake.
+- Create a subdirectory within simdjson, such as `VisualStudio`.
+- Using a shell, go to this newly created directory.
+- Type `cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..` in the shell while in the `VisualStudio` repository. (Alternatively, if you want to build a DLL, you may use the command line `cmake -DCMAKE_GENERATOR_PLATFORM=x64 -DSIMDJSON_BUILD_STATIC=OFF ..`.)
+- This last command created a Visual Studio solution file in the newly created directory (e.g., `simdjson.sln`). Open this file in Visual Studio. You should now be able to build the project and run the tests. For example, in the `Solution Explorer` window (available from the `View` menu), right-click `ALL_BUILD` and select `Build`. To test the code, still in the `Solution Explorer` window, select `RUN_TESTS` and select `Build`.
+
+
+## Usage (Using `vcpkg` on Windows, Linux and MacOS)
+
+[vcpkg](https://github.com/Microsoft/vcpkg) users on Windows, Linux and MacOS can download and install `simdjson` with one single command from their favorite shell.
+
+On Linux and MacOS:
+
+```
+$ ./vcpkg install simdjson
+```
+
+will build and install `simdjson` as a static library.
+
+On Windows (64-bit):
+
+```
+.\vcpkg.exe install simdjson:x64-windows
+```
+
+will build and install `simdjson` as a shared library.
+
+```
+.\vcpkg.exe install simdjson:x64-windows-static  
+```
+
+will build and install `simdjson` as a static library.
+
+These commands will also print out instructions on how to use the library from MSBuild or CMake-based projects.
+
+If you find the version of `simdjson` shipped with `vcpkg` is out-of-date, feel free to report it to `vcpkg` community either by submiting an issue or by creating a PR.


 ## Tools
@@ -219,19 +263,18 @@ The parser builds a useful immutable (read-only) DOM (document-object model) whi
 To simplify the engineering, we make some assumptions.

 - We support UTF-8 (and thus ASCII), nothing else (no Latin, no UTF-16). We do not believe that this is a genuine limitation in the sense that we do not think that there is any serious application that needs to process JSON data without an ASCII or UTF-8 encoding.
- We store strings as NULL terminated C strings. Thus we implicitly assume that you do not include a NULL character within your string, which is allowed technically speaking if you escape it (\u0000).
+- All strings in the JSON document may have up to 4294967295 bytes in UTF-8 (4GB). To enforce this constraint, we refuse to parse a document that contains more than 4294967295 bytes (4GB). This should accomodate most JSON documents.
 - We assume AVX2 support which is available in all recent mainstream x86 processors produced by AMD and Intel. No support for non-x86 processors is included though it can be done. We plan to support ARM processors (help is invited).
 - In cases of failure, we just report a failure without any indication as to the nature of the problem. (This can be easily improved without affecting performance.)
 - As allowed by the specification, we allow repeated keys within an object (other parsers like sajson do the same).
 - Performance is optimized for JSON documents spanning at least a tens kilobytes up to many megabytes: the performance issues with having to parse many tiny JSON documents or one truly enormous JSON document are different.

-*We do not aim to provide a general-purpose JSON library.* A library like RapidJSON offers much more than just parsing, it helps you generate JSON and offers various other convenient functions. We merely parse the document.
-
+_We do not aim to provide a general-purpose JSON library._ A library like RapidJSON offers much more than just parsing, it helps you generate JSON and offers various other convenient functions. We merely parse the document.

 ## Features

 - The input string is unmodified. (Parsers like sajson and RapidJSON use the input string as a buffer.)
- We parse integers and floating-point numbers as separate types which allows us to support large 64-bit integers in [-9223372036854775808,9223372036854775808), like a Java `long` or a C/C++ `long long`. Among the parsers  that differentiate between integers and floating-point numbers, not all support 64-bit integers. (For example, sajson rejects JSON files with integers larger than or equal to 2147483648. RapidJSON will parse a file containing an overly long integer like 18446744073709551616 as a floating-point number.) When we cannot represent exactly an integer as a signed 64-bit value, we reject the JSON document.
+- We parse integers and floating-point numbers as separate types which allows us to support large 64-bit integers in [-9223372036854775808,9223372036854775808), like a Java `long` or a C/C++ `long long`. Among the parsers that differentiate between integers and floating-point numbers, not all support 64-bit integers. (For example, sajson rejects JSON files with integers larger than or equal to 2147483648. RapidJSON will parse a file containing an overly long integer like 18446744073709551616 as a floating-point number.) When we cannot represent exactly an integer as a signed 64-bit value, we reject the JSON document.
 - We do full UTF-8 validation as part of the parsing. (Parsers like fastjson, gason and dropbox json11 do not do UTF-8 validation.)
 - We fully validate the numbers. (Parsers like gason and ultranjson will accept `[0e+]` as valid JSON.)
 - We validate string content for unescaped characters. (Parsers like fastjson and ultrajson accept unescaped line breaks and tabs in strings.)
@@ -243,7 +286,6 @@ The parser works in two stages:
 - Stage 1. (Find marks) Identifies quickly structure elements, strings, and so forth. We validate UTF-8 encoding at that stage.
 - Stage 2. (Structure building) Involves constructing a "tree" of sort (materialized as a tape) to navigate through the data. Strings and numbers are parsed at this stage.

-
 ## Navigating the parsed document

 Here is a code sample to dump back the parsed JSON to a string:
@@ -339,7 +381,6 @@ void simdjson_traverse(std::vector<int64_t> &answer, ParsedJson::iterator &i) {
 }
 ```

-
 ## In-depth comparisons

 If you want to see how a wide range of parsers validate a given JSON file:
@@ -367,8 +408,7 @@ make allparsingcompetition

 - [pysimdjson](https://github.com/TkTech/pysimdjson): Python bindings for the simdjson project.
 - [SimdJsonSharp](https://github.com/EgorBo/SimdJsonSharp): C# version for .NET Core
-
-
+- [simdjson_nodejs](https://github.com/luizperes/simdjson_nodejs): Node.js bindings for the simdjson project.

 ## Various References

@@ -376,7 +416,7 @@ make allparsingcompetition
 - [How to implement atoi using SIMD?](https://stackoverflow.com/questions/35127060/how-to-implement-atoi-using-simd)
 - [Parsing JSON is a Minefield 💣](http://seriot.ch/parsing_json.php)
 - https://tools.ietf.org/html/rfc7159
- The Mison implementation in rust  https://github.com/pikkr/pikkr
+- The Mison implementation in rust https://github.com/pikkr/pikkr
 - http://rapidjson.org/md_doc_sax.html
 - https://github.com/Geal/parser_benchmarks/tree/master/json
 - Gron: A command line tool that makes JSON greppable https://news.ycombinator.com/item?id=16727665
@@ -386,25 +426,25 @@ make allparsingcompetition
 - RapidJSON. http://rapidjson.org/

 Inspiring links:
+
 - https://auth0.com/blog/beating-json-performance-with-protobuf/
 - https://gist.github.com/shijuvar/25ad7de9505232c87034b8359543404a
 - https://github.com/frankmcsherry/blog/blob/master/posts/2018-02-11.md

-
 Validating UTF-8 takes no more than 0.7 cycles per byte:
- https://github.com/lemire/fastvalidate-utf-8 https://lemire.me/blog/2018/05/16/validating-utf-8-strings-using-as-little-as-0-7-cycles-per-byte/

+- https://github.com/lemire/fastvalidate-utf-8 https://lemire.me/blog/2018/05/16/validating-utf-8-strings-using-as-little-as-0-7-cycles-per-byte/

 ## Remarks on JSON parsing

 - The JSON spec defines what a JSON parser is:
->  A JSON parser transforms a JSON text into another representation.  A JSON parser MUST accept all texts that conform to the JSON grammar.  A JSON parser MAY accept non-JSON forms or extensions. An implementation may set limits on the size of texts that it accepts.  An implementation may set limits on the maximum depth of nesting.  An implementation may set limits on the range and precision of numbers.  An implementation may set limits on the length and character contents of strings.
+  > A JSON parser transforms a JSON text into another representation. A JSON parser MUST accept all texts that conform to the JSON grammar. A JSON parser MAY accept non-JSON forms or extensions. An implementation may set limits on the size of texts that it accepts. An implementation may set limits on the maximum depth of nesting. An implementation may set limits on the range and precision of numbers. An implementation may set limits on the length and character contents of strings.

+* JSON is not JavaScript:

- JSON is not JavaScript:
-> All JSON is Javascript but NOT all Javascript is JSON. So {property:1} is invalid because property does not have double quotes around it. {'property':1} is also invalid, because it's single quoted while the only thing that can placate the JSON specification is double quoting. JSON is even fussy enough that {"property":.1} is invalid too, because you should have of course written {"property":0.1}. Also, don't even think about having comments or semicolons, you guessed it: they're invalid. (credit:https://github.com/elzr/vim-json)
+  > All JSON is Javascript but NOT all Javascript is JSON. So {property:1} is invalid because property does not have double quotes around it. {'property':1} is also invalid, because it's single quoted while the only thing that can placate the JSON specification is double quoting. JSON is even fussy enough that {"property":.1} is invalid too, because you should have of course written {"property":0.1}. Also, don't even think about having comments or semicolons, you guessed it: they're invalid. (credit:https://github.com/elzr/vim-json)

- The  structural characters are:
+* The structural characters are:


      begin-array     =  [ left square bracket
@@ -414,7 +454,6 @@ Validating UTF-8 takes no more than 0.7 cycles per byte:
      name-separator  = : colon
      value-separator = , comma

-
 ### Pseudo-structural elements

 A character is pseudo-structural if and only if:
@@ -422,15 +461,12 @@ A character is pseudo-structural if and only if:
 1. Not enclosed in quotes, AND
 2. Is a non-whitespace character, AND
 3. It's preceding character is either:
-(a) a structural character, OR
-(b) whitespace.
+   (a) a structural character, OR
+   (b) whitespace.

 This helps as we redefine some new characters as pseudo-structural such as the characters 1, 1, G, n in the following:

-> { "foo" : 1.5, "bar" : 1.5   GEOFF_IS_A_DUMMY bla bla , "baz", null }
-
-
-
+> { "foo" : 1.5, "bar" : 1.5 GEOFF_IS_A_DUMMY bla bla , "baz", null }

 ## Academic References

@@ -454,8 +490,7 @@ This helps as we redefine some new characters as pseudo-structural such as the c
 - Cameron, Robert D., et al. "Fast Regular Expression Matching with Bit-parallel Data Streams."
 - Lin, Dan. Bits filter: a high-performance multiple string pattern matching algorithm for malware detection. Diss. School of Computing Science-Simon Fraser University, 2010.
 - Yang, Shiyang. Validation of XML Document Based on Parallel Bit Stream Technology. Diss. Applied Sciences: School of Computing Science, 2013.
-  N. Nakasato, "Implementation of a parallel tree method on a GPU", Journal of Computational Science, vol. 3, no. 3, pp. 132-141, 2012.
+- N. Nakasato, "Implementation of a parallel tree method on a GPU", Journal of Computational Science, vol. 3, no. 3, pp. 132-141, 2012.

-
-[license]:LICENSE
-[license img]:https://img.shields.io/badge/License-Apache%202-blue.svg
+[license]: LICENSE
+[license img]: https://img.shields.io/badge/License-Apache%202-blue.svg
@@ -25,6 +25,7 @@ $SCRIPTPATH/src/parsedjsoniterator.cpp
 # order matters
 ALLCHEADERS="
 $SCRIPTPATH/include/simdjson/simdjson_version.h
+$SCRIPTPATH/include/simdjson/simdjson.h
 $SCRIPTPATH/include/simdjson/portability.h
 $SCRIPTPATH/include/simdjson/common_defs.h
 $SCRIPTPATH/include/simdjson/jsoncharutils.h
@@ -54,10 +55,11 @@ function stripinc()
 }
 function dofile()
 {
-    echo "/* begin file $1 */"
+    RELFILE=${1#"$SCRIPTPATH/"}
+    echo "/* begin file $RELFILE */"
    # echo "#line 8 \"$1\"" ## redefining the line/file is not nearly as useful as it sounds for debugging. It breaks IDEs.
    stripinc < $1
-    echo "/* end file $1 */"
+    echo "/* end file $RELFILE */"
 }

 timestamp=$(date)
@@ -1,7 +1,7 @@
 target_include_directories(${SIMDJSON_LIB_NAME}
-  PUBLIC
-  ${PROJECT_SOURCE_DIR}/benchmark
-  ${PROJECT_SOURCE_DIR}/benchmark/linux
+  INTERFACE
+  $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/benchmark>
+  $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/benchmark/linux>
 )

 add_cpp_benchmark(parse)
@@ -171,7 +171,7 @@ int main(int argc, char *argv[]) {
    unified.start();
 #endif

-    isok = isok && unified_machine(p.data(), p.size(), pj);
+    isok = isok && !unified_machine(p.data(), p.size(), pj);
 #ifndef SQUASH_COUNTERS
    unified.end(results);
    cy2 += results[0];
@@ -120,7 +120,7 @@ int main(int argc, char *argv[]) {
  if(!justdata) BEST_TIME("simdjson (dynamic mem) ", build_parsed_json(p).isValid(), true, ,
            repeat, volume, !justdata);
  // (static alloc) 
-  BEST_TIME("simdjson ", json_parse(p, pj), true, , repeat,
+  BEST_TIME("simdjson ", json_parse(p, pj), simdjson::SUCCESS, , repeat,
            volume, !justdata);

 
@@ -5,6 +5,9 @@

 #include <cassert>

+// we support documents up to 4GB
+#define SIMDJSON_MAXSIZE_BYTES 0xFFFFFFFF
+
 // the input buf should be readable up to buf + SIMDJSON_PADDING
 #define SIMDJSON_PADDING  sizeof(__m256i)

@@ -5,87 +5,183 @@
 #include <iomanip>
 #include <iostream>

+// ends with zero char
 static inline void print_with_escapes(const unsigned char *src) {
-  while (*src != 0u) {
+  while (*src) {
    switch (*src) {
-    case '\b':
-      putchar('\\');
-      putchar('b');
-      break;
-    case '\f':
-      putchar('\\');
-      putchar('f');
-      break;
-    case '\n':
-      putchar('\\');
-      putchar('n');
-      break;
-    case '\r':
-      putchar('\\');
-      putchar('r');
-      break;
-    case '\"':
-      putchar('\\');
-      putchar('"');
-      break;
-    case '\t':
-      putchar('\\');
-      putchar('t');
-      break;
-    case '\\':
-      putchar('\\');
-      putchar('\\');
-      break;
-    default:
-      if (*src <= 0x1F) {
-        printf("\\u%04x", *src);
-      } else {
-        putchar(*src);
-}
+      case '\b':
+        putchar('\\');
+        putchar('b');
+        break;
+      case '\f':
+        putchar('\\');
+        putchar('f');
+        break;
+      case '\n':
+        putchar('\\');
+        putchar('n');
+        break;
+      case '\r':
+        putchar('\\');
+        putchar('r');
+        break;
+      case '\"':
+        putchar('\\');
+        putchar('"');
+        break;
+      case '\t':
+        putchar('\\');
+        putchar('t');
+        break;
+      case '\\':
+        putchar('\\');
+        putchar('\\');
+        break;
+      default:
+        if (*src <= 0x1F) {
+          printf("\\u%04x", *src);
+        } else {
+          putchar(*src);
+        }
    }
    src++;
  }
 }

-static inline void print_with_escapes(const unsigned char *src, std::ostream &os) {
-  while (*src != 0u) {
+// ends with zero char
+static inline void print_with_escapes(const unsigned char *src,
+                                      std::ostream &os) {
+  while (*src) {
    switch (*src) {
-    case '\b':
-      os << '\\';
-      os << 'b';
-      break;
-    case '\f':
-      os << '\\';
-      os << 'f';
-      break;
-    case '\n':
-      os << '\\';
-      os << 'n';
-      break;
-    case '\r':
-      os << '\\';
-      os << 'r';
-      break;
-    case '\"':
-      os << '\\';
-      os << '"';
-      break;
-    case '\t':
-      os << '\\';
-      os << 't';
-      break;
-    case '\\':
-      os << '\\';
-      os << '\\';
-      break;
-    default:
-      if (*src <= 0x1F) {
-        std::ios::fmtflags f(os.flags());
-        os << std::hex << std::setw(4) << std::setfill('0') << static_cast<int>(*src);
-        os.flags(f);
-      } else {
-        os << *src;
+      case '\b':
+        os << '\\';
+        os << 'b';
+        break;
+      case '\f':
+        os << '\\';
+        os << 'f';
+        break;
+      case '\n':
+        os << '\\';
+        os << 'n';
+        break;
+      case '\r':
+        os << '\\';
+        os << 'r';
+        break;
+      case '\"':
+        os << '\\';
+        os << '"';
+        break;
+      case '\t':
+        os << '\\';
+        os << 't';
+        break;
+      case '\\':
+        os << '\\';
+        os << '\\';
+        break;
+      default:
+        if (*src <= 0x1F) {
+          std::ios::fmtflags f(os.flags());
+          os << std::hex << std::setw(4) << std::setfill('0')
+             << static_cast<int>(*src);
+          os.flags(f);
+        } else {
+          os << *src;
+        }
+    }
+    src++;
+  }
 }
+
+// print len chars
+static inline void print_with_escapes(const unsigned char *src, size_t len) {
+  const unsigned char *finalsrc = src + len;
+  while (src < finalsrc) {
+    switch (*src) {
+      case '\b':
+        putchar('\\');
+        putchar('b');
+        break;
+      case '\f':
+        putchar('\\');
+        putchar('f');
+        break;
+      case '\n':
+        putchar('\\');
+        putchar('n');
+        break;
+      case '\r':
+        putchar('\\');
+        putchar('r');
+        break;
+      case '\"':
+        putchar('\\');
+        putchar('"');
+        break;
+      case '\t':
+        putchar('\\');
+        putchar('t');
+        break;
+      case '\\':
+        putchar('\\');
+        putchar('\\');
+        break;
+      default:
+        if (*src <= 0x1F) {
+          printf("\\u%04x", *src);
+        } else {
+          putchar(*src);
+        }
+    }
+    src++;
+  }
+}
+
+// print len chars
+static inline void print_with_escapes(const unsigned char *src,
+                                      std::ostream &os, size_t len) {
+  const unsigned char *finalsrc = src + len;
+  while (src < finalsrc) {
+    switch (*src) {
+      case '\b':
+        os << '\\';
+        os << 'b';
+        break;
+      case '\f':
+        os << '\\';
+        os << 'f';
+        break;
+      case '\n':
+        os << '\\';
+        os << 'n';
+        break;
+      case '\r':
+        os << '\\';
+        os << 'r';
+        break;
+      case '\"':
+        os << '\\';
+        os << '"';
+        break;
+      case '\t':
+        os << '\\';
+        os << 't';
+        break;
+      case '\\':
+        os << '\\';
+        os << '\\';
+        break;
+      default:
+        if (*src <= 0x1F) {
+          std::ios::fmtflags f(os.flags());
+          os << std::hex << std::setw(4) << std::setfill('0')
+             << static_cast<int>(*src);
+          os.flags(f);
+        } else {
+          os << *src;
+        }
    }
    src++;
  }
@@ -95,4 +191,10 @@ static inline void print_with_escapes(const char *src, std::ostream &os) {
  print_with_escapes(reinterpret_cast<const unsigned char *>(src), os);
 }

+static inline void print_with_escapes(const char *src, std::ostream &os,
+                                      size_t len) {
+  print_with_escapes(reinterpret_cast<const unsigned char *>(src), os, len);
+}
+
+#
 #endif
@@ -6,23 +6,21 @@
 #include "simdjson/parsedjson.h"
 #include "simdjson/stage1_find_marks.h"
 #include "simdjson/stage2_build_tape.h"
-
-
-
+#include "simdjson/simdjson.h"

 // Parse a document found in buf, need to preallocate ParsedJson.
-// Return false in case of a failure. You can also check validity
-// by calling pj.isValid(). The same ParsedJson can be reused for other documents.
+// Return 0 on success, an error code from simdjson/simdjson.h otherwise
+// You can also check validit by calling pj.isValid(). The same ParsedJson can be reused for other documents.
 //
 // If reallocifneeded is true (default) then a temporary buffer is created when needed during processing
 // (a copy of the input string is made).
 // The input buf should be readable up to buf + len + SIMDJSON_PADDING if reallocifneeded is false,
 // all bytes at and after buf + len  are ignored (can be garbage).
 WARN_UNUSED
-bool json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true);
+int json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true);

 // Parse a document found in buf, need to preallocate ParsedJson.
-// Return false in case of a failure. You can also check validity
+// Return SUCCESS (an integer = 1) in case of a success. You can also check validity
 // by calling pj.isValid(). The same ParsedJson can be reused for other documents.
 //
 // If reallocifneeded is true (default) then a temporary buffer is created when needed during processing
@@ -30,12 +28,12 @@ bool json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifne
 // The input buf should be readable up to buf + len + SIMDJSON_PADDING  if reallocifneeded is false,
 // all bytes at and after buf + len  are ignored (can be garbage).
 WARN_UNUSED
-inline bool json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) {
+inline int json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) {
  return json_parse(reinterpret_cast<const uint8_t *>(buf), len, pj, reallocifneeded);
 }

 // Parse a document found in buf, need to preallocate ParsedJson.
-// Return false in case of a failure. You can also check validity
+// Return SUCCESS (an integer = 1) in case of a success. You can also check validity
 // by calling pj.isValid(). The same ParsedJson can be reused for other documents.
 //
 // If reallocifneeded is true (default) then a temporary buffer is created when needed during processing
@@ -43,7 +41,7 @@ inline bool json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallo
 // the input s should be readable up to s.data() + s.size() + SIMDJSON_PADDING  if reallocifneeded is false,
 // all bytes at and after s.data()+s.size() are ignored (can be garbage).
 WARN_UNUSED
-inline bool json_parse(const std::string_view &s, ParsedJson &pj, bool reallocifneeded = true) {
+inline int json_parse(const std::string_view &s, ParsedJson &pj, bool reallocifneeded = true) {
  return json_parse(s.data(), s.size(), pj, reallocifneeded);
 }

@@ -331,7 +331,7 @@ static never_inline bool parse_large_integer(const uint8_t *const buf,
      return false; // overflow
    }
  }
-  int64_t signed_answer = negative ? -i : i;
+  int64_t signed_answer = negative ? -static_cast<int64_t>(i) : static_cast<int64_t>(i);
  pj.write_tape_s64(signed_answer);
 #ifdef JSON_TEST_NUMBERS // for unit testing
  foundInteger(signed_answer, buf + offset);
@@ -447,11 +447,6 @@ static really_inline bool parse_number(const uint8_t *const buf,
    unsigned char digit = *p - '0';
    expnumber = digit;
    p++;
-    while (is_integer(*p)) {
-      digit = *p - '0';
-      expnumber = 10 * expnumber + digit;
-      ++p;
-    }
    if (is_integer(*p)) {
      digit = *p - '0';
      expnumber = 10 * expnumber + digit;
@@ -30,7 +30,7 @@ public:
  ParsedJson(ParsedJson && p);

  // if needed, allocate memory so that the object is able to process JSON
-  // documents having up to len butes and maxdepth "depth"
+  // documents having up to len bytes and maxdepth "depth"
  WARN_UNUSED
  bool allocateCapacity(size_t len, size_t maxdepth = DEFAULTMAXDEPTH);

@@ -125,8 +125,12 @@ public:
    // get the string value at this node (NULL ended); valid only if we're at "
    // note that tabs, and line endings are escaped in the returned value (see print_with_escapes)
    // return value is valid UTF-8
+    // It may contain NULL chars within the string: get_string_length determines the true 
+    // string length.
    const char * get_string() const;

+    uint32_t get_string_length() const;
+
    // get the double value at this node; valid only if
    // we're at "d"
    double get_double()  const;
@@ -149,6 +153,9 @@ public:
    // if successful, we are left pointing at the value,
    // if not, we are still pointing at the object ({)
    // (in case of repeated keys, this only finds the first one)
+    // We seek the key using C's strcmp so if your JSON strings contain
+    // NULL chars, this would trigger a false positive: if you expect that
+    // to be the case, take extra precautions.
    bool move_to_key(const char * key);

    // throughout return true if we can do the navigation, false
@@ -0,0 +1,16 @@
+#ifndef  SIMDJSON_ERR_H
+# define SIMDJSON_ERR_H
+
+#include <string>
+
+struct simdjson {
+  enum errorValues {
+    SUCCESS = 0,
+    CAPACITY, // This ParsedJson can't support a document that big
+    MEMALLOC, // Error allocating memory, most likely out of memory
+    TAPE_ERROR, // Something went wrong while writing to the tape
+  };
+  static const std::string& errorMsg(const int);
+};
+
+#endif
@@ -1,10 +1,10 @@
 // /include/simdjson/simdjson_version.h automatically generated by release.py, do not change by hand 
 #ifndef SIMDJSON_INCLUDE_SIMDJSON_VERSION 
 #define SIMDJSON_INCLUDE_SIMDJSON_VERSION 
-#define SIMDJSON_VERSION = 0.0.1,  
+#define SIMDJSON_VERSION 0.1.0 
 enum { 
    SIMDJSON_VERSION_MAJOR = 0,  
-    SIMDJSON_VERSION_MINOR = 0,  
-    SIMDJSON_VERSION_REVISION = 1  
+    SIMDJSON_VERSION_MINOR = 1,  
+    SIMDJSON_VERSION_REVISION = 0  
 }; 
 #endif // SIMDJSON_INCLUDE_SIMDJSON_VERSION 
@@ -164,7 +164,7 @@ static inline void avx_count_nibbles(__m256i bytes,

 // check whether the current bytes are valid UTF-8
 // at the end of the function, previous gets updated
-static struct avx_processed_utf_bytes
+static inline struct avx_processed_utf_bytes
 avxcheckUTF8Bytes(__m256i current_bytes,
                  struct avx_processed_utf_bytes *previous,
                  __m256i *has_error) {
@@ -2,14 +2,13 @@
 #define SIMDJSON_STAGE1_FIND_MARKS_H

 #include "simdjson/common_defs.h"
-#include "simdjson/parsedjson.h"
+
+struct ParsedJson;

 WARN_UNUSED
 bool find_structural_bits(const uint8_t *buf, size_t len, ParsedJson &pj);

 WARN_UNUSED
-static inline bool find_structural_bits(const char *buf, size_t len, ParsedJson &pj) {
-    return find_structural_bits(reinterpret_cast<const uint8_t *>(buf), len, pj);
-}
+bool find_structural_bits(const char *buf, size_t len, ParsedJson &pj);

 #endif
@@ -2,17 +2,15 @@
 #define SIMDJSON_STAGE2_BUILD_TAPE_H

 #include "simdjson/common_defs.h"
-#include "simdjson/parsedjson.h"
+
+struct ParsedJson;

 void init_state_machine();

 WARN_UNUSED
-bool unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj);
-
+int unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj);

 WARN_UNUSED
-static inline bool unified_machine(const char *buf, size_t len, ParsedJson &pj) {
-    return unified_machine(reinterpret_cast<const uint8_t *>(buf),len,pj);
-}
+int unified_machine(const char *buf, size_t len, ParsedJson &pj);

 #endif
@@ -79,67 +79,51 @@ really_inline  bool parse_string(const uint8_t *buf, UNUSED size_t len,
  pj.write_tape(0, '"');// don't bother with the string parsing at all
  return true; // always succeeds
 #else
+  pj.write_tape(pj.current_string_buf_loc - pj.string_buf, '"');
  const uint8_t *src = &buf[offset + 1]; // we know that buf at offset is a "
-  uint8_t *dst = pj.current_string_buf_loc;
-#ifdef JSON_TEST_STRINGS // for unit testing
-  uint8_t *const start_of_string = dst;
-#endif
+  uint8_t *dst = pj.current_string_buf_loc + sizeof(uint32_t);
+  const uint8_t *const start_of_string = dst;
  while (1) {
    __m256i v = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(src));
-    auto bs_bits =
-        static_cast<uint32_t>(_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, _mm256_set1_epi8('\\'))));
-    auto quote_bits =
-        static_cast<uint32_t>(_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, _mm256_set1_epi8('"'))));
-#define CHECKUNESCAPED
-    // All Unicode characters may be placed within the
-    // quotation marks, except for the characters that MUST be escaped:
-    // quotation mark, reverse solidus, and the control characters (U+0000
-    //through U+001F).
-    // https://tools.ietf.org/html/rfc8259
-#ifdef CHECKUNESCAPED
-    __m256i unitsep = _mm256_set1_epi8(0x1F);
-    __m256i unescaped_vec = _mm256_cmpeq_epi8(_mm256_max_epu8(unitsep,v),unitsep);// could do it with saturated subtraction
-#endif // CHECKUNESCAPED
-
-    uint32_t quote_dist = trailingzeroes(quote_bits);
-    uint32_t bs_dist = trailingzeroes(bs_bits);
    // store to dest unconditionally - we can overwrite the bits we don't like
    // later
    _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), v);
-    if (quote_dist < bs_dist) {
+    auto bs_bits =
+        static_cast<uint32_t>(_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, _mm256_set1_epi8('\\'))));
+    auto quote_mask = _mm256_cmpeq_epi8(v, _mm256_set1_epi8('"'));
+    auto quote_bits =
+        static_cast<uint32_t>(_mm256_movemask_epi8(quote_mask));
+    if(((bs_bits - 1) & quote_bits) != 0 ) {
      // we encountered quotes first. Move dst to point to quotes and exit
-      dst[quote_dist] = 0; // null terminate and get out

-      pj.write_tape(pj.current_string_buf_loc - pj.string_buf, '"');
+      // find out where the quote is...
+      uint32_t quote_dist = trailingzeroes(quote_bits);
+
+      // NULL termination is still handy if you expect all your strings to be NULL terminated?
+      // It comes at a small cost
+      dst[quote_dist] = 0; 
+
+      uint32_t str_length = (dst - start_of_string) + quote_dist; 
+      memcpy(pj.current_string_buf_loc,&str_length, sizeof(uint32_t));
+      ///////////////////////
+      // Above, check for overflow in case someone has a crazy string (>=4GB?)
+      // But only add the overflow check when the document itself exceeds 4GB
+      // Currently unneeded because we refuse to parse docs larger or equal to 4GB.
+      ////////////////////////
+
+      
+      // we advance the point, accounting for the fact that we have a NULl termination
+      pj.current_string_buf_loc = dst + quote_dist + 1;

-      pj.current_string_buf_loc = dst + quote_dist + 1; // the +1 is due to the 0 value
-#ifdef CHECKUNESCAPED
-      // check that there is no unescaped char before the quote
-      auto unescaped_bits = static_cast<uint32_t>(_mm256_movemask_epi8(unescaped_vec));
-      bool is_ok = ((quote_bits - 1) & (~ quote_bits) & unescaped_bits) == 0;
 #ifdef JSON_TEST_STRINGS // for unit testing
-       if(is_ok) foundString(buf + offset,start_of_string,pj.current_string_buf_loc - 1);
-       else  foundBadString(buf + offset);
-#endif // JSON_TEST_STRINGS
-      return is_ok;
-#else  //CHECKUNESCAPED
-#ifdef JSON_TEST_STRINGS // for unit testing
-       foundString(buf + offset,start_of_string,pj.current_string_buf_loc - 1);
+      foundString(buf + offset,start_of_string,pj.current_string_buf_loc - 1);
 #endif // JSON_TEST_STRINGS
      return true;
-#endif //CHECKUNESCAPED
-    } if (quote_dist > bs_dist) {
+    } 
+    if(((quote_bits - 1) & bs_bits ) != 0 ) {
+      // find out where the backspace is
+      uint32_t bs_dist = trailingzeroes(bs_bits);
      uint8_t escape_char = src[bs_dist + 1];
-#ifdef CHECKUNESCAPED
-      // we are going to need the unescaped_bits to check for unescaped chars
-      auto unescaped_bits = static_cast<uint32_t>(_mm256_movemask_epi8(unescaped_vec));
-      if(((bs_bits - 1) & (~ bs_bits) & unescaped_bits) != 0) {
-#ifdef JSON_TEST_STRINGS // for unit testing
-        foundBadString(buf + offset);
-#endif // JSON_TEST_STRINGS
-        return false;
-      }
-#endif //CHECKUNESCAPED
      // we encountered backslash first. Handle backslash
      if (escape_char == 'u') {
        // move src/dst up to the start; they will be further adjusted
@@ -173,15 +157,6 @@ really_inline  bool parse_string(const uint8_t *buf, UNUSED size_t len,
      // neither.
      src += 32;
      dst += 32;
-#ifdef CHECKUNESCAPED
-      // check for unescaped chars
-      if(_mm256_testz_si256(unescaped_vec,unescaped_vec) != 1) {
-#ifdef JSON_TEST_STRINGS // for unit testing
-          foundBadString(buf + offset);
-#endif // JSON_TEST_STRINGS
-        return false;
-      }
-#endif // CHECKUNESCAPED
    }
  }
  // can't be reached
@@ -0,0 +1 @@
+[1e+1111]
@@ -1,4 +1,4 @@
-/* auto-generated on Tue 26 Feb 2019 10:14:31 EST. Do not edit! */
+/* auto-generated on Wed 13 Mar 2019 20:02:04 EDT. Do not edit! */

 #include <iostream>
 #include "simdjson.h"
@@ -1,17 +1,35 @@
-/* auto-generated on Tue 26 Feb 2019 10:14:31 EST. Do not edit! */
-/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/simdjson_version.h */
+/* auto-generated on Wed 13 Mar 2019 20:02:04 EDT. Do not edit! */
+/* begin file include/simdjson/simdjson_version.h */
 // /include/simdjson/simdjson_version.h automatically generated by release.py, do not change by hand 
 #ifndef SIMDJSON_INCLUDE_SIMDJSON_VERSION 
 #define SIMDJSON_INCLUDE_SIMDJSON_VERSION 
-#define SIMDJSON_VERSION = 0.0.1,  
+#define SIMDJSON_VERSION 0.1.0 
 enum { 
    SIMDJSON_VERSION_MAJOR = 0,  
-    SIMDJSON_VERSION_MINOR = 0,  
-    SIMDJSON_VERSION_REVISION = 1  
+    SIMDJSON_VERSION_MINOR = 1,  
+    SIMDJSON_VERSION_REVISION = 0  
 }; 
 #endif // SIMDJSON_INCLUDE_SIMDJSON_VERSION 
-/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/simdjson_version.h */
-/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/portability.h */
+/* end file include/simdjson/simdjson_version.h */
+/* begin file include/simdjson/simdjson.h */
+#ifndef  SIMDJSON_ERR_H
+# define SIMDJSON_ERR_H
+
+#include <string>
+
+struct simdjson {
+  enum errorValues {
+    SUCCESS = 0,
+    CAPACITY, // This ParsedJson can't support a document that big
+    MEMALLOC, // Error allocating memory, most likely out of memory
+    TAPE_ERROR, // Something went wrong while writing to the tape
+  };
+  static const std::string& errorMsg(const int);
+};
+
+#endif
+/* end file include/simdjson/simdjson.h */
+/* begin file include/simdjson/portability.h */
 #ifndef SIMDJSON_PORTABILITY_H
 #define SIMDJSON_PORTABILITY_H

@@ -139,8 +157,8 @@ static inline void aligned_free(void *memblock) {
 }

 #endif // SIMDJSON_PORTABILITY_H
-/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/portability.h */
-/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/common_defs.h */
+/* end file include/simdjson/portability.h */
+/* begin file include/simdjson/common_defs.h */
 #ifndef SIMDJSON_COMMON_DEFS_H
 #define SIMDJSON_COMMON_DEFS_H

@@ -199,8 +217,8 @@ static inline void aligned_free(void *memblock) {
 #endif  // MSC_VER

 #endif // SIMDJSON_COMMON_DEFS_H
-/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/common_defs.h */
-/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/jsoncharutils.h */
+/* end file include/simdjson/common_defs.h */
+/* begin file include/simdjson/jsoncharutils.h */
 #ifndef SIMDJSON_JSONCHARUTILS_H
 #define SIMDJSON_JSONCHARUTILS_H

@@ -322,8 +340,8 @@ inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) {
 }

 #endif
-/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/jsoncharutils.h */
-/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/jsonformatutils.h */
+/* end file include/simdjson/jsoncharutils.h */
+/* begin file include/simdjson/jsonformatutils.h */
 #ifndef SIMDJSON_JSONFORMATUTILS_H
 #define SIMDJSON_JSONFORMATUTILS_H

@@ -422,8 +440,8 @@ static inline void print_with_escapes(const char *src, std::ostream &os) {
 }

 #endif
-/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/jsonformatutils.h */
-/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/jsonioutil.h */
+/* end file include/simdjson/jsonformatutils.h */
+/* begin file include/simdjson/jsonioutil.h */
 #ifndef SIMDJSON_JSONIOUTIL_H
 #define SIMDJSON_JSONIOUTIL_H

@@ -463,8 +481,8 @@ std::string_view  get_corpus(const std::string& filename);


 #endif
-/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/jsonioutil.h */
-/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/simdprune_tables.h */
+/* end file include/simdjson/jsonioutil.h */
+/* begin file include/simdjson/simdprune_tables.h */
 #ifndef SIMDJSON_SIMDPRUNE_TABLES_H
 #define SIMDJSON_SIMDPRUNE_TABLES_H

@@ -35543,8 +35561,8 @@ static const uint32_t mask256_epi32[] = {
 #endif //__AVX2__

 #endif
-/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/simdprune_tables.h */
-/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/simdutf8check.h */
+/* end file include/simdjson/simdprune_tables.h */
+/* begin file include/simdjson/simdutf8check.h */

 #ifndef SIMDJSON_SIMDUTF8CHECK_H
 #define SIMDJSON_SIMDUTF8CHECK_H
@@ -35710,7 +35728,7 @@ static inline void avx_count_nibbles(__m256i bytes,

 // check whether the current bytes are valid UTF-8
 // at the end of the function, previous gets updated
-static struct avx_processed_utf_bytes
+static inline struct avx_processed_utf_bytes
 avxcheckUTF8Bytes(__m256i current_bytes,
                  struct avx_processed_utf_bytes *previous,
                  __m256i *has_error) {
@@ -35739,8 +35757,8 @@ avxcheckUTF8Bytes(__m256i current_bytes,
 #warning "We require AVX2 support!"
 #endif // __AVX2__
 #endif
-/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/simdutf8check.h */
-/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/jsonminifier.h */
+/* end file include/simdjson/simdutf8check.h */
+/* begin file include/simdjson/jsonminifier.h */
 #ifndef SIMDJSON_JSONMINIFIER_H
 #define SIMDJSON_JSONMINIFIER_H

@@ -35763,8 +35781,8 @@ static inline size_t jsonminify(const std::string_view & p, char *out) {
 }

 #endif
-/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/jsonminifier.h */
-/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/parsedjson.h */
+/* end file include/simdjson/jsonminifier.h */
+/* begin file include/simdjson/parsedjson.h */
 #ifndef SIMDJSON_PARSEDJSON_H
 #define SIMDJSON_PARSEDJSON_H

@@ -35794,7 +35812,7 @@ public:
  ParsedJson(ParsedJson && p);

  // if needed, allocate memory so that the object is able to process JSON
-  // documents having up to len butes and maxdepth "depth"
+  // documents having up to len bytes and maxdepth "depth"
  WARN_UNUSED
  bool allocateCapacity(size_t len, size_t maxdepth = DEFAULTMAXDEPTH);

@@ -36016,23 +36034,23 @@ inline void dumpbits32_always(uint32_t v, const std::string &msg) {


 #endif
-/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/parsedjson.h */
-/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/stage1_find_marks.h */
+/* end file include/simdjson/parsedjson.h */
+/* begin file include/simdjson/stage1_find_marks.h */
 #ifndef SIMDJSON_STAGE1_FIND_MARKS_H
 #define SIMDJSON_STAGE1_FIND_MARKS_H


+struct ParsedJson;
+
 WARN_UNUSED
 bool find_structural_bits(const uint8_t *buf, size_t len, ParsedJson &pj);

 WARN_UNUSED
-static inline bool find_structural_bits(const char *buf, size_t len, ParsedJson &pj) {
-    return find_structural_bits(reinterpret_cast<const uint8_t *>(buf), len, pj);
-}
+bool find_structural_bits(const char *buf, size_t len, ParsedJson &pj);

 #endif
-/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/stage1_find_marks.h */
-/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/stringparsing.h */
+/* end file include/simdjson/stage1_find_marks.h */
+/* begin file include/simdjson/stringparsing.h */
 #ifndef SIMDJSON_STRINGPARSING_H
 #define SIMDJSON_STRINGPARSING_H

@@ -36223,8 +36241,8 @@ really_inline  bool parse_string(const uint8_t *buf, UNUSED size_t len,


 #endif
-/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/stringparsing.h */
-/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/numberparsing.h */
+/* end file include/simdjson/stringparsing.h */
+/* begin file include/simdjson/numberparsing.h */
 #ifndef SIMDJSON_NUMBERPARSING_H
 #define SIMDJSON_NUMBERPARSING_H

@@ -36554,7 +36572,7 @@ static never_inline bool parse_large_integer(const uint8_t *const buf,
      return false; // overflow
    }
  }
-  int64_t signed_answer = negative ? -i : i;
+  int64_t signed_answer = negative ? -static_cast<int64_t>(i) : static_cast<int64_t>(i);
  pj.write_tape_s64(signed_answer);
 #ifdef JSON_TEST_NUMBERS // for unit testing
  foundInteger(signed_answer, buf + offset);
@@ -36670,11 +36688,6 @@ static really_inline bool parse_number(const uint8_t *const buf,
    unsigned char digit = *p - '0';
    expnumber = digit;
    p++;
-    while (is_integer(*p)) {
-      digit = *p - '0';
-      expnumber = 10 * expnumber + digit;
-      ++p;
-    }
    if (is_integer(*p)) {
      digit = *p - '0';
      expnumber = 10 * expnumber + digit;
@@ -36741,43 +36754,39 @@ static really_inline bool parse_number(const uint8_t *const buf,
 }

 #endif
-/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/numberparsing.h */
-/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/stage2_build_tape.h */
+/* end file include/simdjson/numberparsing.h */
+/* begin file include/simdjson/stage2_build_tape.h */
 #ifndef SIMDJSON_STAGE2_BUILD_TAPE_H
 #define SIMDJSON_STAGE2_BUILD_TAPE_H


+struct ParsedJson;
+
 void init_state_machine();

 WARN_UNUSED
-bool unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj);
-
+int unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj);

 WARN_UNUSED
-static inline bool unified_machine(const char *buf, size_t len, ParsedJson &pj) {
-    return unified_machine(reinterpret_cast<const uint8_t *>(buf),len,pj);
-}
+int unified_machine(const char *buf, size_t len, ParsedJson &pj);

 #endif
-/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/stage2_build_tape.h */
-/* begin file /Users/lemire/CVS/github/simdjson/include/simdjson/jsonparser.h */
+/* end file include/simdjson/stage2_build_tape.h */
+/* begin file include/simdjson/jsonparser.h */
 #ifndef SIMDJSON_JSONPARSER_H
 #define SIMDJSON_JSONPARSER_H


-
-
-
 // Parse a document found in buf, need to preallocate ParsedJson.
-// Return false in case of a failure. You can also check validity
-// by calling pj.isValid(). The same ParsedJson can be reused for other documents.
+// Return 0 on success, an error code from simdjson/simdjson.h otherwise
+// You can also check validit by calling pj.isValid(). The same ParsedJson can be reused for other documents.
 //
 // If reallocifneeded is true (default) then a temporary buffer is created when needed during processing
 // (a copy of the input string is made).
 // The input buf should be readable up to buf + len + SIMDJSON_PADDING if reallocifneeded is false,
 // all bytes at and after buf + len  are ignored (can be garbage).
 WARN_UNUSED
-bool json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true);
+int json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded = true);

 // Parse a document found in buf, need to preallocate ParsedJson.
 // Return false in case of a failure. You can also check validity
@@ -36788,7 +36797,7 @@ bool json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifne
 // The input buf should be readable up to buf + len + SIMDJSON_PADDING  if reallocifneeded is false,
 // all bytes at and after buf + len  are ignored (can be garbage).
 WARN_UNUSED
-inline bool json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) {
+inline int json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallocifneeded = true) {
  return json_parse(reinterpret_cast<const uint8_t *>(buf), len, pj, reallocifneeded);
 }

@@ -36801,7 +36810,7 @@ inline bool json_parse(const char * buf, size_t len, ParsedJson &pj, bool reallo
 // the input s should be readable up to s.data() + s.size() + SIMDJSON_PADDING  if reallocifneeded is false,
 // all bytes at and after s.data()+s.size() are ignored (can be garbage).
 WARN_UNUSED
-inline bool json_parse(const std::string_view &s, ParsedJson &pj, bool reallocifneeded = true) {
+inline int json_parse(const std::string_view &s, ParsedJson &pj, bool reallocifneeded = true) {
  return json_parse(s.data(), s.size(), pj, reallocifneeded);
 }

@@ -36840,4 +36849,4 @@ inline ParsedJson build_parsed_json(const std::string_view &s, bool reallocifnee
 }

 #endif
-/* end file /Users/lemire/CVS/github/simdjson/include/simdjson/jsonparser.h */
+/* end file include/simdjson/jsonparser.h */
@@ -3,6 +3,7 @@ if(${CMAKE_C_COMPILER_ID} MATCHES "Intel") # icc / icpc
  set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-intel")
 endif()

+include(GNUInstallDirs)

 # we default on a shared library.
 if(SIMDJSON_BUILD_STATIC)
@@ -21,20 +22,38 @@ set(SIMDJSON_SRC
    stage1_find_marks.cpp
    stage2_build_tape.cpp
    parsedjson.cpp
-    parsedjsoniterator.cpp)
+    parsedjsoniterator.cpp
+    simdjson.cpp
+    )
         
 add_library(${SIMDJSON_LIB_NAME} ${SIMDJSON_LIB_TYPE} ${SIMDJSON_SRC})
+
 target_include_directories(${SIMDJSON_LIB_NAME}
-  PUBLIC ${PROJECT_SOURCE_DIR}/include
+  PUBLIC
+  $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
 )

-install(TARGETS ${SIMDJSON_LIB_NAME} DESTINATION lib)
+install(TARGETS ${SIMDJSON_LIB_NAME}
+  EXPORT ${SIMDJSON_LIB_NAME}-config
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+)
+
+install(EXPORT ${SIMDJSON_LIB_NAME}-config
+  FILE ${SIMDJSON_LIB_NAME}-config.cmake
+  NAMESPACE ${SIMDJSON_LIB_NAME}::
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${SIMDJSON_LIB_NAME}
+)

 if(NOT MSVC)
 ## We output the library at the root of the current directory where cmake is invoked
 ## This is handy but Visual Studio will happily ignore us
 set_target_properties(${SIMDJSON_LIB_NAME} PROPERTIES
-  LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
+  LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
+  VERSION ${SIMDJSON_LIB_VERSION}
+  SOVERSION ${SIMDJSON_LIB_SOVERSION})
 MESSAGE( STATUS "Library output directory (does not apply to Visual Studio): " ${CMAKE_BINARY_DIR})
 endif()

@@ -6,17 +6,9 @@ char * allocate_padded_buffer(size_t length) {
    // we could do a simple malloc
    //return (char *) malloc(length + SIMDJSON_PADDING);
    // However, we might as well align to cache lines...
-    char *padded_buffer;
    size_t totalpaddedlength = length + SIMDJSON_PADDING;
-#ifdef _MSC_VER
-	padded_buffer = (char*) _aligned_malloc(totalpaddedlength, 64);
-#elif defined(__MINGW32__) || defined(__MINGW64__)
-	padded_buffer = __mingw_aligned_malloc(totalpaddedlength, 64);
-#else
-    if (posix_memalign(reinterpret_cast<void **>(&padded_buffer), 64, totalpaddedlength) != 0) { return nullptr;
-}
-#endif
-	return padded_buffer;
+    char *padded_buffer = (char *) aligned_malloc(64, totalpaddedlength);
+    return padded_buffer;
 }

 std::string_view get_corpus(const std::string& filename) {
@@ -5,21 +5,13 @@
 #else
 #include <unistd.h>
 #endif
-
-
-
-
-
-
-
+#include "simdjson/simdjson.h"

 // parse a document found in buf, need to preallocate ParsedJson.
 WARN_UNUSED
-bool json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded) {
+int json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifneeded) {
  if (pj.bytecapacity < len) {
-    std::cerr << "Your ParsedJson cannot support documents that big: " << len
-              << std::endl;
-    return false;
+    return simdjson::CAPACITY;
  }
  bool reallocated = false;
  if(reallocifneeded) {
@@ -33,24 +25,19 @@ bool json_parse(const uint8_t *buf, size_t len, ParsedJson &pj, bool reallocifne
 #endif
 	 if ( (reinterpret_cast<uintptr_t>(buf + len - 1) % pagesize ) < SIMDJSON_PADDING ) {
       const uint8_t *tmpbuf  = buf;
-       buf = reinterpret_cast<uint8_t *>(allocate_padded_buffer(len));
-       if(buf == nullptr) { return false;
-}
+       buf = (uint8_t *) allocate_padded_buffer(len);
+       if(buf == NULL) return simdjson::MEMALLOC;
       memcpy((void*)buf,tmpbuf,len);
       reallocated = true;
     }
  }
-  bool isok = find_structural_bits(buf, len, pj);
-  if (isok) {
-    isok = unified_machine(buf, len, pj);
-  } else {
-    if(reallocated) { free((void*)buf);
-}
-    return false;
+  // find_structural_bits returns a boolean, not an int, we invert its result to keep consistent with res == 0 meaning success
+  int res = !find_structural_bits(buf, len, pj);
+  if (!res) {
+    res = unified_machine(buf, len, pj);
  }
-  if(reallocated) { free((void*)buf);
-}
-  return isok;
+  if(reallocated) { aligned_free((void*)buf);}
+  return res;
 }

 WARN_UNUSED
@@ -58,7 +45,8 @@ ParsedJson build_parsed_json(const uint8_t *buf, size_t len, bool reallocifneede
  ParsedJson pj;
  bool ok = pj.allocateCapacity(len);
  if(ok) {
-    ok = json_parse(buf, len, pj, reallocifneeded);
+    int res = json_parse(buf, len, pj, reallocifneeded);
+    ok = res == simdjson::SUCCESS;
    assert(ok == pj.isValid());
  } else {
    std::cerr << "failure during memory allocation " << std::endl;
@@ -38,19 +38,22 @@ bool ParsedJson::allocateCapacity(size_t len, size_t maxdepth) {
      std::cerr << "capacities must be non-zero " << std::endl;
      return false;
    }
-    if (len > 0) {
-      if ((len <= bytecapacity) && (depthcapacity < maxdepth)) {
-        return true;
-}
-      deallocate();
+    if(len > SIMDJSON_MAXSIZE_BYTES) {
+      return false;
    }
+    if ((len <= bytecapacity) && (depthcapacity < maxdepth)) {
+      return true;
+    }
+    deallocate();
    isvalid = false;
    bytecapacity = 0; // will only set it to len after allocations are a success
    n_structural_indexes = 0;
    uint32_t max_structures = ROUNDUP_N(len, 64) + 2 + 7;
    structural_indexes = new (std::nothrow) uint32_t[max_structures];
    size_t localtapecapacity = ROUNDUP_N(len, 64);
-    size_t localstringcapacity = ROUNDUP_N(len + 32, 64);
+    // a document with only zero-length strings... could have len/3 string
+    // and we would need len/3 * 5 bytes on the string buffer 
+    size_t localstringcapacity = ROUNDUP_N(5 * len / 3 + 32, 64); 
    string_buf = new (std::nothrow) uint8_t[localstringcapacity];
    tape = new (std::nothrow) uint64_t[localtapecapacity];
    containing_scope_offset = new (std::nothrow) uint32_t[maxdepth];
@@ -87,16 +90,11 @@ void ParsedJson::deallocate() {
    depthcapacity = 0;
    tapecapacity = 0;
    stringcapacity = 0;
-     {delete[] ret_address;
-}
-     {delete[] containing_scope_offset;
-}
-     {delete[] tape;
-}
-     {delete[] string_buf;
-}
-     {delete[] structural_indexes;
-}
+    delete[] ret_address;
+    delete[] containing_scope_offset;
+    delete[] tape;
+    delete[] string_buf;
+    delete[] structural_indexes;
    isvalid = false;
 }

@@ -108,8 +106,10 @@ void ParsedJson::init() {

 WARN_UNUSED
 bool ParsedJson::printjson(std::ostream &os) {
-    if(!isvalid) { return false;
-}
+    if(!isvalid) { 
+      return false;
+    }
+    uint32_t string_length;
    size_t tapeidx = 0;
    uint64_t tape_val = tape[tapeidx];
    uint8_t type = (tape_val >> 56);
@@ -138,34 +138,39 @@ bool ParsedJson::printjson(std::ostream &os) {
      if (!inobject[depth]) {
        if ((inobjectidx[depth] > 0) && (type != ']')) {
          os << ",";
-}
+        }
        inobjectidx[depth]++;
      } else { // if (inobject) {
        if ((inobjectidx[depth] > 0) && ((inobjectidx[depth] & 1) == 0) &&
            (type != '}')) {
          os << ",";
-}
+        }
        if (((inobjectidx[depth] & 1) == 1)) {
          os << ":";
-}
+        }
        inobjectidx[depth]++;
      }
      switch (type) {
      case '"': // we have a string
        os << '"';
-        print_with_escapes((const unsigned char *)(string_buf + payload));
+        memcpy(&string_length,string_buf + payload, sizeof(uint32_t));
+        print_with_escapes((const unsigned char *)(string_buf + payload + sizeof(uint32_t)), string_length); 
        os << '"';
        break;
      case 'l': // we have a long int
        if (tapeidx + 1 >= howmany) {
+          delete[] inobject;
+          delete[] inobjectidx;
          return false;
-}
+        }
        os <<  static_cast<int64_t>(tape[++tapeidx]);
        break;
      case 'd': // we have a double
-        if (tapeidx + 1 >= howmany) {
+        if (tapeidx + 1 >= howmany){
+          delete[] inobject;
+          delete[] inobjectidx;
          return false;
-}
+        }
        double answer;
        memcpy(&answer, &tape[++tapeidx], sizeof(answer));
        os << answer;
@@ -218,8 +223,10 @@ bool ParsedJson::printjson(std::ostream &os) {

 WARN_UNUSED
 bool ParsedJson::dump_raw_tape(std::ostream &os) {
-    if(!isvalid) { return false;
-}
+    if(!isvalid) { 
+      return false;
+    }
+    uint32_t string_length;
    size_t tapeidx = 0;
    uint64_t tape_val = tape[tapeidx];
    uint8_t type = (tape_val >> 56);
@@ -242,21 +249,22 @@ bool ParsedJson::dump_raw_tape(std::ostream &os) {
      switch (type) {
      case '"': // we have a string
        os << "string \"";
-        print_with_escapes((const unsigned char *)(string_buf + payload));
+        memcpy(&string_length,string_buf + payload, sizeof(uint32_t));
+        print_with_escapes((const unsigned char *)(string_buf + payload + sizeof(uint32_t)), string_length);
        os << '"';
        os << '\n';
        break;
      case 'l': // we have a long int
        if (tapeidx + 1 >= howmany) {
          return false;
-}
+        }
        os << "integer " << static_cast<int64_t>(tape[++tapeidx]) << "\n";
        break;
      case 'd': // we have a double
        os << "float ";
        if (tapeidx + 1 >= howmany) {
          return false;
-}
+        }
        double answer;
        memcpy(&answer, &tape[++tapeidx], sizeof(answer));
        os << answer << '\n';
@@ -1,5 +1,6 @@
 #include "simdjson/parsedjson.h"
 #include "simdjson/common_defs.h"
+#include <iterator>

 ParsedJson::iterator::iterator(ParsedJson &pj_) : pj(pj_), depth(0), location(0), tape_length(0), depthindex(nullptr) {
        if(pj.isValid()) {
@@ -73,31 +74,30 @@ uint8_t ParsedJson::iterator::get_scope_type() const {
 }

 bool ParsedJson::iterator::move_forward() {
-    if(location + 1 >= tape_length) {
-    return false; // we are at the end!
-    }
-    // we are entering a new scope
-    if ((current_type == '[') || (current_type == '{')){
-    depth++;
-    depthindex[depth].start_of_scope = location;
-    depthindex[depth].scope_type = current_type;
-    }
-    location = location + 1;
-    current_val = pj.tape[location];
-    current_type = (current_val >> 56);
-    // if we encounter a scope closure, we need to move up
-    while ((current_type == ']') || (current_type == '}')) {
    if(location + 1 >= tape_length) {
        return false; // we are at the end!
    }
-    depth--;
-    if(depth == 0) {
-        return false; // should not be necessary
+
+    if ((current_type == '[') || (current_type == '{')){
+        // We are entering a new scope
+        depth++;
+        depthindex[depth].start_of_scope = location;
+        depthindex[depth].scope_type = current_type;
+    } else if ((current_type == ']') || (current_type == '}')) {
+        // Leaving a scope.
+        depth--;
+        if(depth == 0) {
+            // Should not be necessary
+            return false;
+        }
+    } else if ((current_type == 'd') || (current_type == 'l')) {
+        // d and l types use 2 locations on the tape, not just one.
+        location += 1;
    }
-    location = location + 1;
+
+    location += 1;
    current_val = pj.tape[location];
    current_type = (current_val >> 56);
-    }
    return true;
 }

@@ -107,24 +107,32 @@ uint8_t ParsedJson::iterator::get_type()  const {


 int64_t ParsedJson::iterator::get_integer()  const {
-    if(location + 1 >= tape_length) { return 0;// default value in case of error
-}
+    if(location + 1 >= tape_length) { 
+      return 0;// default value in case of error
+    }
    return static_cast<int64_t>(pj.tape[location + 1]);
 }

 double ParsedJson::iterator::get_double()  const {
-    if(location + 1 >= tape_length) { return NAN;// default value in case of error
-}
+    if(location + 1 >= tape_length) { 
+      return NAN;// default value in case of error
+    }
    double answer;
    memcpy(&answer, & pj.tape[location + 1], sizeof(answer));
    return answer;
 }

 const char * ParsedJson::iterator::get_string() const {
-    return  reinterpret_cast<const char *>(pj.string_buf + (current_val & JSONVALUEMASK)) ;
+   return  reinterpret_cast<const char *>(pj.string_buf + (current_val & JSONVALUEMASK) + sizeof(uint32_t)) ;
 }


+uint32_t ParsedJson::iterator::get_string_length() const {
+    uint32_t answer;
+    memcpy(&answer, reinterpret_cast<const char *>(pj.string_buf + (current_val & JSONVALUEMASK)), sizeof(uint32_t));
+    return answer;
+}
+
 bool ParsedJson::iterator::is_object_or_array() const {
    return is_object_or_array(get_type());
 }
@@ -155,14 +163,15 @@ bool ParsedJson::iterator::is_object_or_array(uint8_t type) {

 bool ParsedJson::iterator::move_to_key(const char * key) {
    if(down()) {
-    do {
+      do {
        assert(is_string());
-        bool rightkey = (strcmp(get_string(),key)==0);
+        bool rightkey = (strcmp(get_string(),key)==0);// null chars would fool this
        next();
-        if(rightkey) { return true;
-}
-    } while(next());
-    assert(up());// not found
+        if(rightkey) { 
+          return true;
+        }
+      } while(next());
+      assert(up());// not found
    }
    return false;
 }
@@ -261,15 +270,17 @@ void ParsedJson::iterator::to_start_scope()  {
 }

 bool ParsedJson::iterator::print(std::ostream &os, bool escape_strings) const {
-    if(!isOk()) { return false;
-}
+    if(!isOk()) { 
+      return false;
+    }
    switch (current_type) {
    case '"': // we have a string
    os << '"';
    if(escape_strings) {
-        print_with_escapes(get_string(), os);
+        print_with_escapes(get_string(), os, get_string_length());
    } else {
-        os << get_string();
+        // was: os << get_string();, but given that we can include null chars, we have to do something crazier:
+        std::copy(get_string(), get_string() + get_string_length(), std::ostream_iterator<char>(os));
    }
    os << '"';
    break;
@@ -0,0 +1,13 @@
+#include <map>
+#include "simdjson/simdjson.h"
+
+const std::map<int, const std::string> errorStrings = {
+    {simdjson::SUCCESS, "No errors"},
+    {simdjson::CAPACITY, "This ParsedJson can't support a document that big"},
+    {simdjson::MEMALLOC, "Error allocating memory, we're most likely out of memory"},
+    {simdjson::TAPE_ERROR, "Something went wrong while writing to the tape"}
+};
+
+const std::string& simdjson::errorMsg(const int errorCode) {
+    return errorStrings.at(errorCode);
+}
@@ -1,7 +1,7 @@
-#include "simdjson/portability.h"
+#include <cassert>
 #include "simdjson/common_defs.h"
 #include "simdjson/parsedjson.h"
-#include <cassert>
+#include "simdjson/portability.h"

 #ifndef SIMDJSON_SKIPUTF8VALIDATION
 #define SIMDJSON_UTF8VALIDATE
@@ -15,11 +15,29 @@
 #endif
 using namespace std;

+really_inline void check_utf8(__m256i input_lo, __m256i input_hi,
+                              __m256i &has_error,
+                              struct avx_processed_utf_bytes &previous) {
+  __m256i highbit = _mm256_set1_epi8(0x80);
+  if ((_mm256_testz_si256(_mm256_or_si256(input_lo, input_hi), highbit)) == 1) {
+    // it is ascii, we just check continuation
+    has_error = _mm256_or_si256(
+        _mm256_cmpgt_epi8(
+            previous.carried_continuations,
+            _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
+                             9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)),
+        has_error);
+  } else {
+    // it is not ascii so we have to do heavy work
+    previous = avxcheckUTF8Bytes(input_lo, &previous, &has_error);
+    previous = avxcheckUTF8Bytes(input_hi, &previous, &has_error);
+  }
+}

 // a straightforward comparison of a mask against input. 5 uops; would be
 // cheaper in AVX512.
-really_inline uint64_t cmp_mask_against_input(__m256i input_lo, __m256i input_hi,
-                                         __m256i mask) {
+really_inline uint64_t cmp_mask_against_input(__m256i input_lo,
+                                              __m256i input_hi, __m256i mask) {
  __m256i cmp_res_0 = _mm256_cmpeq_epi8(input_lo, mask);
  uint64_t res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(cmp_res_0));
  __m256i cmp_res_1 = _mm256_cmpeq_epi8(input_hi, mask);
@@ -27,212 +45,299 @@ really_inline uint64_t cmp_mask_against_input(__m256i input_lo, __m256i input_hi
  return res_0 | (res_1 << 32);
 }

+// find all values less than or equal than the content of maxval (using unsigned arithmetic) 
+really_inline uint64_t unsigned_lteq_against_input(__m256i input_lo,
+                                              __m256i input_hi, __m256i maxval) {
+  __m256i cmp_res_0 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval,input_lo),maxval);
+  uint64_t res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(cmp_res_0));
+  __m256i cmp_res_1 = _mm256_cmpeq_epi8(_mm256_max_epu8(maxval,input_hi),maxval);
+  uint64_t res_1 = _mm256_movemask_epi8(cmp_res_1);
+  return res_0 | (res_1 << 32);
+}
+
+// return a bitvector indicating where we have characters that end an odd-length
+// sequence of backslashes (and thus change the behavior of the next character
+// to follow). A even-length sequence of backslashes, and, for that matter, the
+// largest even-length prefix of our odd-length sequence of backslashes, simply
+// modify the behavior of the backslashes themselves.
+// We also update the prev_iter_ends_odd_backslash reference parameter to
+// indicate whether we end an iteration on an odd-length sequence of
+// backslashes, which modifies our subsequent search for odd-length
+// sequences of backslashes in an obvious way.
+really_inline uint64_t
+find_odd_backslash_sequences(__m256i input_lo, __m256i input_hi,
+                             uint64_t &prev_iter_ends_odd_backslash) {
+  const uint64_t even_bits = 0x5555555555555555ULL;
+  const uint64_t odd_bits = ~even_bits;
+  uint64_t bs_bits =
+      cmp_mask_against_input(input_lo, input_hi, _mm256_set1_epi8('\\'));
+  uint64_t start_edges = bs_bits & ~(bs_bits << 1);
+  // flip lowest if we have an odd-length run at the end of the prior
+  // iteration
+  uint64_t even_start_mask = even_bits ^ prev_iter_ends_odd_backslash;
+  uint64_t even_starts = start_edges & even_start_mask;
+  uint64_t odd_starts = start_edges & ~even_start_mask;
+  uint64_t even_carries = bs_bits + even_starts;
+
+  uint64_t odd_carries;
+  // must record the carry-out of our odd-carries out of bit 63; this
+  // indicates whether the sense of any edge going to the next iteration
+  // should be flipped
+  bool iter_ends_odd_backslash =
+      add_overflow(bs_bits, odd_starts, &odd_carries);
+
+  odd_carries |=
+      prev_iter_ends_odd_backslash;  // push in bit zero as a potential end
+                                     // if we had an odd-numbered run at the
+                                     // end of the previous iteration
+  prev_iter_ends_odd_backslash = iter_ends_odd_backslash ? 0x1ULL : 0x0ULL;
+  uint64_t even_carry_ends = even_carries & ~bs_bits;
+  uint64_t odd_carry_ends = odd_carries & ~bs_bits;
+  uint64_t even_start_odd_end = even_carry_ends & odd_bits;
+  uint64_t odd_start_even_end = odd_carry_ends & even_bits;
+  uint64_t odd_ends = even_start_odd_end | odd_start_even_end;
+  return odd_ends;
+}
+
+// return both the quote mask (which is a half-open mask that covers the first
+// quote
+// in an unescaped quote pair and everything in the quote pair) and the quote
+// bits, which are the simple
+// unescaped quoted bits. We also update the prev_iter_inside_quote value to
+// tell the next iteration
+// whether we finished the final iteration inside a quote pair; if so, this
+// inverts our behavior of
+// whether we're inside quotes for the next iteration.
+// Note that we don't do any error checking to see if we have backslash
+// sequences outside quotes; these
+// backslash sequences (of any length) will be detected elsewhere.
+really_inline uint64_t find_quote_mask_and_bits(
+    __m256i input_lo, __m256i input_hi, uint64_t odd_ends,
+    uint64_t &prev_iter_inside_quote, uint64_t &quote_bits, uint64_t &error_mask) {
+  quote_bits =
+      cmp_mask_against_input(input_lo, input_hi, _mm256_set1_epi8('"'));
+  quote_bits = quote_bits & ~odd_ends;
+  // remove from the valid quoted region the unescapted characters.
+  uint64_t quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128(
+      _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFF), 0));
+  quote_mask ^= prev_iter_inside_quote;
+  // All Unicode characters may be placed within the
+  // quotation marks, except for the characters that MUST be escaped:
+  // quotation mark, reverse solidus, and the control characters (U+0000
+  //through U+001F).
+  // https://tools.ietf.org/html/rfc8259
+  uint64_t unescaped = unsigned_lteq_against_input(input_lo, input_hi, _mm256_set1_epi8(0x1F));
+  error_mask |= quote_mask & unescaped;
+  // right shift of a signed value expected to be well-defined and standard
+  // compliant as of C++20,
+  // John Regher from Utah U. says this is fine code
+  prev_iter_inside_quote =
+      static_cast<uint64_t>(static_cast<int64_t>(quote_mask) >> 63);
+  return quote_mask;
+}
+
+really_inline void find_whitespace_and_structurals(const __m256i input_lo,
+                                                   __m256i input_hi,
+                                                   uint64_t &whitespace,
+                                                   uint64_t &structurals) {
+  // do a 'shufti' to detect structural JSON characters
+  // they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c
+  // these go into the first 3 buckets of the comparison (1/2/4)
+
+  // we are also interested in the four whitespace characters
+  // space 0x20, linefeed 0x0a, horizontal tab 0x09 and carriage return 0x0d
+  // these go into the next 2 buckets of the comparison (8/16)
+  const __m256i low_nibble_mask = _mm256_setr_epi8(
+      16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0,
+      0, 8, 12, 1, 2, 9, 0, 0);
+  const __m256i high_nibble_mask = _mm256_setr_epi8(
+      8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0, 8, 0, 18, 4, 0, 1, 0, 1,
+      0, 0, 0, 3, 2, 1, 0, 0);
+
+  __m256i structural_shufti_mask = _mm256_set1_epi8(0x7);
+  __m256i whitespace_shufti_mask = _mm256_set1_epi8(0x18);
+
+  __m256i v_lo = _mm256_and_si256(
+      _mm256_shuffle_epi8(low_nibble_mask, input_lo),
+      _mm256_shuffle_epi8(high_nibble_mask,
+                          _mm256_and_si256(_mm256_srli_epi32(input_lo, 4),
+                                           _mm256_set1_epi8(0x7f))));
+
+  __m256i v_hi = _mm256_and_si256(
+      _mm256_shuffle_epi8(low_nibble_mask, input_hi),
+      _mm256_shuffle_epi8(high_nibble_mask,
+                          _mm256_and_si256(_mm256_srli_epi32(input_hi, 4),
+                                           _mm256_set1_epi8(0x7f))));
+  __m256i tmp_lo = _mm256_cmpeq_epi8(
+      _mm256_and_si256(v_lo, structural_shufti_mask), _mm256_set1_epi8(0));
+  __m256i tmp_hi = _mm256_cmpeq_epi8(
+      _mm256_and_si256(v_hi, structural_shufti_mask), _mm256_set1_epi8(0));
+
+  uint64_t structural_res_0 =
+      static_cast<uint32_t>(_mm256_movemask_epi8(tmp_lo));
+  uint64_t structural_res_1 = _mm256_movemask_epi8(tmp_hi);
+  structurals = ~(structural_res_0 | (structural_res_1 << 32));
+
+  __m256i tmp_ws_lo = _mm256_cmpeq_epi8(
+      _mm256_and_si256(v_lo, whitespace_shufti_mask), _mm256_set1_epi8(0));
+  __m256i tmp_ws_hi = _mm256_cmpeq_epi8(
+      _mm256_and_si256(v_hi, whitespace_shufti_mask), _mm256_set1_epi8(0));
+
+  uint64_t ws_res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(tmp_ws_lo));
+  uint64_t ws_res_1 = _mm256_movemask_epi8(tmp_ws_hi);
+  whitespace = ~(ws_res_0 | (ws_res_1 << 32));
+}
+
+// flatten out values in 'bits' assuming that they are are to have values of idx
+// plus their position in the bitvector, and store these indexes at
+// base_ptr[base] incrementing base as we go
+// will potentially store extra values beyond end of valid bits, so base_ptr
+// needs to be large enough to handle this
+really_inline void flatten_bits(uint32_t *base_ptr, uint32_t &base,
+                                uint32_t idx, uint64_t bits) {
+  uint32_t cnt = hamming(bits);
+  uint32_t next_base = base + cnt;
+  while (bits != 0u) {
+    base_ptr[base + 0] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[base + 1] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[base + 2] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[base + 3] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[base + 4] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[base + 5] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[base + 6] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(bits);
+    bits = bits & (bits - 1);
+    base_ptr[base + 7] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(bits);
+    bits = bits & (bits - 1);
+    base += 8;
+  }
+  base = next_base;
+}
+
+// return a updated structural bit vector with quoted contents cleared out and
+// pseudo-structural characters added to the mask
+// updates prev_iter_ends_pseudo_pred which tells us whether the previous
+// iteration ended on a whitespace or a structural character (which means that
+// the next iteration
+// will have a pseudo-structural character at its start)
+really_inline uint64_t finalize_structurals(
+    uint64_t structurals, uint64_t whitespace, uint64_t quote_mask,
+    uint64_t quote_bits, uint64_t &prev_iter_ends_pseudo_pred) {
+  // mask off anything inside quotes
+  structurals &= ~quote_mask;
+  // add the real quote bits back into our bitmask as well, so we can
+  // quickly traverse the strings we've spent all this trouble gathering
+  structurals |= quote_bits;
+  // Now, establish "pseudo-structural characters". These are non-whitespace
+  // characters that are (a) outside quotes and (b) have a predecessor that's
+  // either whitespace or a structural character. This means that subsequent
+  // passes will get a chance to encounter the first character of every string
+  // of non-whitespace and, if we're parsing an atom like true/false/null or a
+  // number we can stop at the first whitespace or structural character
+  // following it.
+
+  // a qualified predecessor is something that can happen 1 position before an
+  // psuedo-structural character
+  uint64_t pseudo_pred = structurals | whitespace;
+
+  uint64_t shifted_pseudo_pred =
+      (pseudo_pred << 1) | prev_iter_ends_pseudo_pred;
+  prev_iter_ends_pseudo_pred = pseudo_pred >> 63;
+  uint64_t pseudo_structurals =
+      shifted_pseudo_pred & (~whitespace) & (~quote_mask);
+  structurals |= pseudo_structurals;
+
+  // now, we've used our close quotes all we need to. So let's switch them off
+  // they will be off in the quote mask and on in quote bits.
+  structurals &= ~(quote_bits & ~quote_mask);
+  return structurals;
+}
+
 WARN_UNUSED
 /*never_inline*/ bool find_structural_bits(const uint8_t *buf, size_t len,
                                           ParsedJson &pj) {
  if (len > pj.bytecapacity) {
-    cerr << "Your ParsedJson object only supports documents up to "<< pj.bytecapacity << " bytes but you are trying to process " <<  len  << " bytes\n";
+    cerr << "Your ParsedJson object only supports documents up to "
+         << pj.bytecapacity << " bytes but you are trying to process " << len
+         << " bytes\n";
    return false;
  }
  uint32_t *base_ptr = pj.structural_indexes;
  uint32_t base = 0;
 #ifdef SIMDJSON_UTF8VALIDATE
  __m256i has_error = _mm256_setzero_si256();
-  struct avx_processed_utf_bytes previous{};
+  struct avx_processed_utf_bytes previous {};
  previous.rawbytes = _mm256_setzero_si256();
  previous.high_nibbles = _mm256_setzero_si256();
  previous.carried_continuations = _mm256_setzero_si256();
- #endif
+#endif

-  // Useful constant masks
-  const uint64_t even_bits = 0x5555555555555555ULL;
-  const uint64_t odd_bits = ~even_bits;
-
-  // for now, just work in 64-byte chunks
  // we have padded the input out to 64 byte multiple with the remainder being
  // zeros

  // persistent state across loop
-  uint64_t prev_iter_ends_odd_backslash = 0ULL; // either 0 or 1, but a 64-bit value
-  uint64_t prev_iter_inside_quote = 0ULL;       // either all zeros or all ones
-
-  // effectively the very first char is considered to follow "whitespace" for the
-  // purposes of psuedo-structural character detection
+  // does the last iteration end with an odd-length sequence of backslashes? 
+  // either 0 or 1, but a 64-bit value
+  uint64_t prev_iter_ends_odd_backslash = 0ULL;
+  // does the previous iteration end inside a double-quote pair?
+  uint64_t prev_iter_inside_quote = 0ULL;  // either all zeros or all ones
+  // does the previous iteration end on something that is a predecessor of a
+  // pseudo-structural character - i.e. whitespace or a structural character
+  // effectively the very first char is considered to follow "whitespace" for
+  // the
+  // purposes of pseudo-structural character detection so we initialize to 1
  uint64_t prev_iter_ends_pseudo_pred = 1ULL;
+
+  // structurals are persistent state across loop as we flatten them on the
+  // subsequent iteration into our array pointed to be base_ptr.
+  // This is harmless on the first iteration as structurals==0
+  // and is done for performance reasons; we can hide some of the latency of the
+  // expensive carryless multiply in the previous step with this work
+  uint64_t structurals = 0;
+
  size_t lenminus64 = len < 64 ? 0 : len - 64;
  size_t idx = 0;
-  uint64_t structurals = 0;
+  uint64_t error_mask = 0; // for unescaped characters within strings (ASCII code points < 0x20)
+
  for (; idx < lenminus64; idx += 64) {
 #ifndef _MSC_VER
    __builtin_prefetch(buf + idx + 128);
 #endif
-    __m256i input_lo = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(buf + idx + 0));
-    __m256i input_hi = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(buf + idx + 32));
+    __m256i input_lo =
+        _mm256_loadu_si256(reinterpret_cast<const __m256i *>(buf + idx + 0));
+    __m256i input_hi =
+        _mm256_loadu_si256(reinterpret_cast<const __m256i *>(buf + idx + 32));
+
 #ifdef SIMDJSON_UTF8VALIDATE
-    __m256i highbit = _mm256_set1_epi8(0x80);
-    if((_mm256_testz_si256(_mm256_or_si256(input_lo, input_hi),highbit)) == 1) {
-        // it is ascii, we just check continuation
-        has_error = _mm256_or_si256(
-          _mm256_cmpgt_epi8(previous.carried_continuations,
-                          _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
-                                           9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
-                                           9, 9, 9, 9, 9, 9, 9, 1)),has_error);
-
-    } else {
-        // it is not ascii so we have to do heavy work
-        previous = avxcheckUTF8Bytes(input_lo, &previous, &has_error);
-        previous = avxcheckUTF8Bytes(input_hi, &previous, &has_error);
-    }
+    check_utf8(input_lo, input_hi, has_error, previous);
 #endif
-    ////////////////////////////////////////////////////////////////////////////////////////////
-    //     Step 1: detect odd sequences of backslashes
-    ////////////////////////////////////////////////////////////////////////////////////////////

-    uint64_t bs_bits =
-        cmp_mask_against_input(input_lo, input_hi, _mm256_set1_epi8('\\'));
-    uint64_t start_edges = bs_bits & ~(bs_bits << 1);
-    // flip lowest if we have an odd-length run at the end of the prior
-    // iteration
-    uint64_t even_start_mask = even_bits ^ prev_iter_ends_odd_backslash;
-    uint64_t even_starts = start_edges & even_start_mask;
-    uint64_t odd_starts = start_edges & ~even_start_mask;
-    uint64_t even_carries = bs_bits + even_starts;
+    // detect odd sequences of backslashes
+    uint64_t odd_ends = find_odd_backslash_sequences(
+        input_lo, input_hi, prev_iter_ends_odd_backslash);

-    uint64_t odd_carries;
-    // must record the carry-out of our odd-carries out of bit 63; this
-    // indicates whether the sense of any edge going to the next iteration
-    // should be flipped
-    bool iter_ends_odd_backslash =
-		add_overflow(bs_bits, odd_starts, &odd_carries);
+    // detect insides of quote pairs ("quote_mask") and also our quote_bits
+    // themselves
+    uint64_t quote_bits;
+    uint64_t quote_mask = find_quote_mask_and_bits(
+        input_lo, input_hi, odd_ends, prev_iter_inside_quote, quote_bits, error_mask);

-    odd_carries |=
-        prev_iter_ends_odd_backslash; // push in bit zero as a potential end
-                                      // if we had an odd-numbered run at the
-                                      // end of the previous iteration
-    prev_iter_ends_odd_backslash = iter_ends_odd_backslash ? 0x1ULL : 0x0ULL;
-    uint64_t even_carry_ends = even_carries & ~bs_bits;
-    uint64_t odd_carry_ends = odd_carries & ~bs_bits;
-    uint64_t even_start_odd_end = even_carry_ends & odd_bits;
-    uint64_t odd_start_even_end = odd_carry_ends & even_bits;
-    uint64_t odd_ends = even_start_odd_end | odd_start_even_end;
+    // take the previous iterations structural bits, not our current iteration,
+    // and flatten
+    flatten_bits(base_ptr, base, idx, structurals);

-    ////////////////////////////////////////////////////////////////////////////////////////////
-    //     Step 2: detect insides of quote pairs
-    ////////////////////////////////////////////////////////////////////////////////////////////
+    uint64_t whitespace;
+    find_whitespace_and_structurals(input_lo, input_hi, whitespace,
+                                    structurals);

-    uint64_t quote_bits =
-        cmp_mask_against_input(input_lo, input_hi, _mm256_set1_epi8('"'));
-    quote_bits = quote_bits & ~odd_ends;
-    uint64_t quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128(
-        _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFF), 0));
-
-
-
-    uint32_t cnt = hamming(structurals);
-    uint32_t next_base = base + cnt;
-    while (structurals != 0u) {
-      base_ptr[base + 0] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 1] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 2] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 3] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 4] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 5] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 6] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                   
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 7] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals); 
-      structurals = structurals & (structurals - 1);
-      base += 8;
-    }
-    base = next_base;
-
-    quote_mask ^= prev_iter_inside_quote;
-    prev_iter_inside_quote = static_cast<uint64_t>(static_cast<int64_t>(quote_mask) >> 63); // right shift of a signed value expected to be well-defined and standard compliant as of C++20, John Regher from Utah U. says this is fine code
-
-    // How do we build up a user traversable data structure
-    // first, do a 'shufti' to detect structural JSON characters
-    // they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c
-    // these go into the first 3 buckets of the comparison (1/2/4)
-
-    // we are also interested in the four whitespace characters
-    // space 0x20, linefeed 0x0a, horizontal tab 0x09 and carriage return 0x0d
-    // these go into the next 2 buckets of the comparison (8/16)
-    const __m256i low_nibble_mask = _mm256_setr_epi8(
-        //  0                           9  a   b  c  d
-        16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0, 16, 0, 0, 0, 0, 0, 0,
-        0, 0, 8, 12, 1, 2, 9, 0, 0);
-    const __m256i high_nibble_mask = _mm256_setr_epi8(
-        //  0     2   3     5     7
-        8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0, 8, 0, 18, 4, 0, 1, 0,
-        1, 0, 0, 0, 3, 2, 1, 0, 0);
-
-    __m256i structural_shufti_mask = _mm256_set1_epi8(0x7);
-    __m256i whitespace_shufti_mask = _mm256_set1_epi8(0x18);
-
-    __m256i v_lo = _mm256_and_si256(
-        _mm256_shuffle_epi8(low_nibble_mask, input_lo),
-        _mm256_shuffle_epi8(high_nibble_mask,
-                            _mm256_and_si256(_mm256_srli_epi32(input_lo, 4),
-                                             _mm256_set1_epi8(0x7f))));
-
-    __m256i v_hi = _mm256_and_si256(
-        _mm256_shuffle_epi8(low_nibble_mask, input_hi),
-        _mm256_shuffle_epi8(high_nibble_mask,
-                            _mm256_and_si256(_mm256_srli_epi32(input_hi, 4),
-                                             _mm256_set1_epi8(0x7f))));
-    __m256i tmp_lo = _mm256_cmpeq_epi8(
-        _mm256_and_si256(v_lo, structural_shufti_mask), _mm256_set1_epi8(0));
-    __m256i tmp_hi = _mm256_cmpeq_epi8(
-        _mm256_and_si256(v_hi, structural_shufti_mask), _mm256_set1_epi8(0));
-
-    uint64_t structural_res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(tmp_lo));
-    uint64_t structural_res_1 = _mm256_movemask_epi8(tmp_hi);
-    structurals = ~(structural_res_0 | (structural_res_1 << 32));
-
-    // this additional mask and transfer is non-trivially expensive,
-    // unfortunately
-    __m256i tmp_ws_lo = _mm256_cmpeq_epi8(
-        _mm256_and_si256(v_lo, whitespace_shufti_mask), _mm256_set1_epi8(0));
-    __m256i tmp_ws_hi = _mm256_cmpeq_epi8(
-        _mm256_and_si256(v_hi, whitespace_shufti_mask), _mm256_set1_epi8(0));
-
-    uint64_t ws_res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(tmp_ws_lo));
-    uint64_t ws_res_1 = _mm256_movemask_epi8(tmp_ws_hi);
-    uint64_t whitespace = ~(ws_res_0 | (ws_res_1 << 32));
-    // mask off anything inside quotes
-    structurals &= ~quote_mask;
-
-    // add the real quote bits back into our bitmask as well, so we can
-    // quickly traverse the strings we've spent all this trouble gathering
-    structurals |= quote_bits;
-
-    // Now, establish "pseudo-structural characters". These are non-whitespace
-    // characters that are (a) outside quotes and (b) have a predecessor that's
-    // either whitespace or a structural character. This means that subsequent
-    // passes will get a chance to encounter the first character of every string
-    // of non-whitespace and, if we're parsing an atom like true/false/null or a
-    // number we can stop at the first whitespace or structural character
-    // following it.
-
-    // a qualified predecessor is something that can happen 1 position before an
-    // psuedo-structural character
-    uint64_t pseudo_pred = structurals | whitespace;
-    uint64_t shifted_pseudo_pred = (pseudo_pred << 1) | prev_iter_ends_pseudo_pred;
-    prev_iter_ends_pseudo_pred = pseudo_pred >> 63;
-    uint64_t pseudo_structurals =
-        shifted_pseudo_pred & (~whitespace) & (~quote_mask);
-    structurals |= pseudo_structurals;
-
-    // now, we've used our close quotes all we need to. So let's switch them off
-    // they will be off in the quote mask and on in quote bits.
-    structurals &= ~(quote_bits & ~quote_mask);
-
-    //*(uint64_t *)(pj.structurals + idx / 8) = structurals;
+    // fixup structurals to reflect quotes and add pseudo-structural characters
+    structurals = finalize_structurals(structurals, whitespace, quote_mask,
+                                       quote_bits, prev_iter_ends_pseudo_pred);
  }

  ////////////////
@@ -242,215 +347,70 @@ WARN_UNUSED
  ////////////
  if (idx < len) {
    uint8_t tmpbuf[64];
-    memset(tmpbuf,0x20,64);
-    memcpy(tmpbuf,buf+idx,len - idx);
-    __m256i input_lo = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(tmpbuf + 0));
-    __m256i input_hi = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(tmpbuf + 32));
+    memset(tmpbuf, 0x20, 64);
+    memcpy(tmpbuf, buf + idx, len - idx);
+    __m256i input_lo =
+        _mm256_loadu_si256(reinterpret_cast<const __m256i *>(tmpbuf + 0));
+    __m256i input_hi =
+        _mm256_loadu_si256(reinterpret_cast<const __m256i *>(tmpbuf + 32));
+
 #ifdef SIMDJSON_UTF8VALIDATE
-    __m256i highbit = _mm256_set1_epi8(0x80);
-    if((_mm256_testz_si256(_mm256_or_si256(input_lo, input_hi),highbit)) == 1) {
-        // it is ascii, we just check continuation
-        has_error = _mm256_or_si256(
-          _mm256_cmpgt_epi8(previous.carried_continuations,
-                          _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
-                                           9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
-                                           9, 9, 9, 9, 9, 9, 9, 1)),has_error);
-
-    } else {
-        // it is not ascii so we have to do heavy work
-        previous = avxcheckUTF8Bytes(input_lo, &previous, &has_error);
-        previous = avxcheckUTF8Bytes(input_hi, &previous, &has_error);
-    }
+    check_utf8(input_lo, input_hi, has_error, previous);
 #endif
-    ////////////////////////////////////////////////////////////////////////////////////////////
-    //     Step 1: detect odd sequences of backslashes
-    ////////////////////////////////////////////////////////////////////////////////////////////

-    uint64_t bs_bits =
-        cmp_mask_against_input(input_lo, input_hi, _mm256_set1_epi8('\\'));
-    uint64_t start_edges = bs_bits & ~(bs_bits << 1);
-    // flip lowest if we have an odd-length run at the end of the prior
-    // iteration
-    uint64_t even_start_mask = even_bits ^ prev_iter_ends_odd_backslash;
-    uint64_t even_starts = start_edges & even_start_mask;
-    uint64_t odd_starts = start_edges & ~even_start_mask;
-    uint64_t even_carries = bs_bits + even_starts;
+    // detect odd sequences of backslashes
+    uint64_t odd_ends = find_odd_backslash_sequences(
+        input_lo, input_hi, prev_iter_ends_odd_backslash);

-    uint64_t odd_carries;
-    // must record the carry-out of our odd-carries out of bit 63; this
-    // indicates whether the sense of any edge going to the next iteration
-    // should be flipped
-    //bool iter_ends_odd_backslash =
-	add_overflow(bs_bits, odd_starts, &odd_carries);
+    // detect insides of quote pairs ("quote_mask") and also our quote_bits
+    // themselves
+    uint64_t quote_bits;
+    uint64_t quote_mask = find_quote_mask_and_bits(
+        input_lo, input_hi, odd_ends, prev_iter_inside_quote, quote_bits, error_mask);

-    odd_carries |=
-        prev_iter_ends_odd_backslash; // push in bit zero as a potential end
-                                      // if we had an odd-numbered run at the
-                                      // end of the previous iteration
-    //prev_iter_ends_odd_backslash = iter_ends_odd_backslash ? 0x1ULL : 0x0ULL;
-    uint64_t even_carry_ends = even_carries & ~bs_bits;
-    uint64_t odd_carry_ends = odd_carries & ~bs_bits;
-    uint64_t even_start_odd_end = even_carry_ends & odd_bits;
-    uint64_t odd_start_even_end = odd_carry_ends & even_bits;
-    uint64_t odd_ends = even_start_odd_end | odd_start_even_end;
+    // take the previous iterations structural bits, not our current iteration,
+    // and flatten
+    flatten_bits(base_ptr, base, idx, structurals);

-    ////////////////////////////////////////////////////////////////////////////////////////////
-    //     Step 2: detect insides of quote pairs
-    ////////////////////////////////////////////////////////////////////////////////////////////
+    uint64_t whitespace;
+    find_whitespace_and_structurals(input_lo, input_hi, whitespace,
+                                    structurals);

-    uint64_t quote_bits =
-        cmp_mask_against_input(input_lo, input_hi, _mm256_set1_epi8('"'));
-    quote_bits = quote_bits & ~odd_ends;
-    uint64_t quote_mask = _mm_cvtsi128_si64(_mm_clmulepi64_si128(
-        _mm_set_epi64x(0ULL, quote_bits), _mm_set1_epi8(0xFF), 0));
-    quote_mask ^= prev_iter_inside_quote;
-    //prev_iter_inside_quote = (uint64_t)((int64_t)quote_mask >> 63); // right shift of a signed value expected to be well-defined and standard compliant as of C++20
-
-    uint32_t cnt = hamming(structurals);
-    uint32_t next_base = base + cnt;
-    while (structurals != 0u) {
-      base_ptr[base + 0] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 1] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 2] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 3] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 4] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 5] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 6] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 7] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base += 8;
-    }
-    base = next_base;
-    // How do we build up a user traversable data structure
-    // first, do a 'shufti' to detect structural JSON characters
-    // they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c
-    // these go into the first 3 buckets of the comparison (1/2/4)
-
-    // we are also interested in the four whitespace characters
-    // space 0x20, linefeed 0x0a, horizontal tab 0x09 and carriage return 0x0d
-    // these go into the next 2 buckets of the comparison (8/16)
-    const __m256i low_nibble_mask = _mm256_setr_epi8(
-        //  0                           9  a   b  c  d
-        16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0, 16, 0, 0, 0, 0, 0, 0,
-        0, 0, 8, 12, 1, 2, 9, 0, 0);
-    const __m256i high_nibble_mask = _mm256_setr_epi8(
-        //  0     2   3     5     7
-        8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0, 8, 0, 18, 4, 0, 1, 0,
-        1, 0, 0, 0, 3, 2, 1, 0, 0);
-
-    __m256i structural_shufti_mask = _mm256_set1_epi8(0x7);
-    __m256i whitespace_shufti_mask = _mm256_set1_epi8(0x18);
-
-    __m256i v_lo = _mm256_and_si256(
-        _mm256_shuffle_epi8(low_nibble_mask, input_lo),
-        _mm256_shuffle_epi8(high_nibble_mask,
-                            _mm256_and_si256(_mm256_srli_epi32(input_lo, 4),
-                                             _mm256_set1_epi8(0x7f))));
-
-    __m256i v_hi = _mm256_and_si256(
-        _mm256_shuffle_epi8(low_nibble_mask, input_hi),
-        _mm256_shuffle_epi8(high_nibble_mask,
-                            _mm256_and_si256(_mm256_srli_epi32(input_hi, 4),
-                                             _mm256_set1_epi8(0x7f))));
-    __m256i tmp_lo = _mm256_cmpeq_epi8(
-        _mm256_and_si256(v_lo, structural_shufti_mask), _mm256_set1_epi8(0));
-    __m256i tmp_hi = _mm256_cmpeq_epi8(
-        _mm256_and_si256(v_hi, structural_shufti_mask), _mm256_set1_epi8(0));
-
-    uint64_t structural_res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(tmp_lo));
-    uint64_t structural_res_1 = _mm256_movemask_epi8(tmp_hi);
-    structurals = ~(structural_res_0 | (structural_res_1 << 32));
-
-    // this additional mask and transfer is non-trivially expensive,
-    // unfortunately
-    __m256i tmp_ws_lo = _mm256_cmpeq_epi8(
-        _mm256_and_si256(v_lo, whitespace_shufti_mask), _mm256_set1_epi8(0));
-    __m256i tmp_ws_hi = _mm256_cmpeq_epi8(
-        _mm256_and_si256(v_hi, whitespace_shufti_mask), _mm256_set1_epi8(0));
-
-    uint64_t ws_res_0 = static_cast<uint32_t>(_mm256_movemask_epi8(tmp_ws_lo));
-    uint64_t ws_res_1 = _mm256_movemask_epi8(tmp_ws_hi);
-    uint64_t whitespace = ~(ws_res_0 | (ws_res_1 << 32));
-
-
-    // mask off anything inside quotes
-    structurals &= ~quote_mask;
-
-    // add the real quote bits back into our bitmask as well, so we can
-    // quickly traverse the strings we've spent all this trouble gathering
-    structurals |= quote_bits;
-
-    // Now, establish "pseudo-structural characters". These are non-whitespace
-    // characters that are (a) outside quotes and (b) have a predecessor that's
-    // either whitespace or a structural character. This means that subsequent
-    // passes will get a chance to encounter the first character of every string
-    // of non-whitespace and, if we're parsing an atom like true/false/null or a
-    // number we can stop at the first whitespace or structural character
-    // following it.
-
-    // a qualified predecessor is something that can happen 1 position before an
-    // psuedo-structural character
-    uint64_t pseudo_pred = structurals | whitespace;
-    uint64_t shifted_pseudo_pred = (pseudo_pred << 1) | prev_iter_ends_pseudo_pred;
-    //prev_iter_ends_pseudo_pred = pseudo_pred >> 63;
-    uint64_t pseudo_structurals =
-        shifted_pseudo_pred & (~whitespace) & (~quote_mask);
-    structurals |= pseudo_structurals;
-
-    // now, we've used our close quotes all we need to. So let's switch them off
-    // they will be off in the quote mask and on in quote bits.
-    structurals &= ~(quote_bits & ~quote_mask);
-    //*(uint64_t *)(pj.structurals + idx / 8) = structurals;
+    // fixup structurals to reflect quotes and add pseudo-structural characters
+    structurals = finalize_structurals(structurals, whitespace, quote_mask,
+                                       quote_bits, prev_iter_ends_pseudo_pred);
    idx += 64;
  }
-    uint32_t cnt = hamming(structurals);
-    uint32_t next_base = base + cnt;
-    while (structurals != 0u) {
-      base_ptr[base + 0] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 1] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 2] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 3] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 4] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 5] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 6] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base_ptr[base + 7] = static_cast<uint32_t>(idx) - 64 + trailingzeroes(structurals);                          
-      structurals = structurals & (structurals - 1);
-      base += 8;
-    }
-    base = next_base;
+  // finally, flatten out the remaining structurals from the last iteration
+  flatten_bits(base_ptr, base, idx, structurals);

  pj.n_structural_indexes = base;
-  // a valid JSON file cannot have zero structural indexes - we should have found something
+  // a valid JSON file cannot have zero structural indexes - we should have
+  // found something
  if (pj.n_structural_indexes == 0u) {
    return false;
  }
-  if(base_ptr[pj.n_structural_indexes-1] > len) {
-    fprintf( stderr,"Internal bug\n");
+  if (base_ptr[pj.n_structural_indexes - 1] > len) {
+    fprintf(stderr, "Internal bug\n");
    return false;
  }
-  if(len != base_ptr[pj.n_structural_indexes-1]) {
-    // the string might not be NULL terminated, but we add a virtual NULL ending character. 
+  if (len != base_ptr[pj.n_structural_indexes - 1]) {
+    // the string might not be NULL terminated, but we add a virtual NULL ending
+    // character.
    base_ptr[pj.n_structural_indexes++] = len;
  }
-  base_ptr[pj.n_structural_indexes] = 0; // make it safe to dereference one beyond this array
-
+  // make it safe to dereference one beyond this array
+  base_ptr[pj.n_structural_indexes] = 0;  
+  if (error_mask) {
+    return false;
+  }
 #ifdef SIMDJSON_UTF8VALIDATE
  return _mm256_testz_si256(has_error, has_error) != 0;
 #else
  return true;
 #endif
 }
+
+bool find_structural_bits(const char *buf, size_t len, ParsedJson &pj) {
+  return find_structural_bits(reinterpret_cast<const uint8_t *>(buf), len, pj);
+}
@@ -13,6 +13,7 @@
 #include "simdjson/numberparsing.h"
 #include "simdjson/parsedjson.h"
 #include "simdjson/stringparsing.h"
+#include "simdjson/simdjson.h"

 #include <iostream>
 #define PATH_SEP '/'
@@ -62,7 +63,7 @@ really_inline bool is_valid_null_atom(const uint8_t *loc) {
 * for documentation.
 ***********/
 WARN_UNUSED
-bool unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) {
+int unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) {
  uint32_t i = 0; // index of the structural character (0,1,2,3...)
  uint32_t idx;   // location of the structural character in the input (buf)
  uint8_t c; // used to track the (structural) character we are looking at, updated
@@ -70,8 +71,7 @@ bool unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj) {
  uint32_t depth = 0; // could have an arbitrary starting depth
  pj.init();
  if(pj.bytecapacity < len) {
-      fprintf(stderr, "insufficient capacity\n");
-      return false;
+      return simdjson::CAPACITY;
  }
 // this macro reads the next structural character, updating idx, i and c.
 #define UPDATE_CHAR()                                                          \
@@ -509,8 +509,12 @@ succeed:


  pj.isvalid  = true;
-  return true;
+  return simdjson::SUCCESS;

 fail:
-  return false;
+  return simdjson::TAPE_ERROR;
+}
+
+int unified_machine(const char *buf, size_t len, ParsedJson &pj) {
+  return unified_machine(reinterpret_cast<const uint8_t*>(buf), len, pj);
 }
@@ -103,6 +103,7 @@ Hint: we can read the first tape element to determine the length of the tape.


 ## Strings
+We prefix the string data itself by a 32-bit header to be interpreted as a 32-bit integer. It indicates the length of the string. The actual string data starts at an offset of 4 bytes.

 We store string values using UTF-8 encoding with null termination on a separate tape. A string value is represented on the main tape as the 64-bit tape element `('"'<< 56) + x` where the payload `x` is the location on the string tape of the null-terminated string. 

@@ -1,8 +1,12 @@
 if(MSVC)
-target_include_directories(${SIMDJSON_LIB_NAME}
-  PUBLIC ${PROJECT_SOURCE_DIR}/windows
-)
+  target_include_directories(${SIMDJSON_LIB_NAME}
+    INTERFACE $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/windows>
+  )
 endif()

 add_cpp_test(jsoncheck)
-add_test(jsoncheck jsoncheck)
+
+add_executable(singleheader ./singleheadertest.cpp ${PROJECT_SOURCE_DIR}/singleheader/simdjson.cpp)
+target_compile_definitions(singleheader PRIVATE JSON_TEST_PATH="${PROJECT_SOURCE_DIR}/jsonexamples/twitter.json")
+target_link_libraries(singleheader ${SIMDJSON_LIB_NAME})
+add_test(singleheader singleheader)
@@ -79,24 +79,20 @@ bool validate(const char *dirname) {
        return false;
      }
      ++howmany;
-      bool isok = json_parse(p, pj);
+      const int parseRes = json_parse(p, pj);
      aligned_free((void*)p.data());
-      printf("%s\n", isok ? "ok" : "invalid");
+      printf("%s\n", parseRes == 0 ? "ok" : "invalid");
      if(contains("EXCLUDE",name)) {
        // skipping
        howmany--;
-      } else if (startsWith("pass", name)) {
-        if (!isok) {
+      } else if (startsWith("pass", name) && parseRes != 0) {
          isfileasexpected[i] = false;
-          printf("warning: file %s should pass but it fails.\n", name);
+          printf("warning: file %s should pass but it fails. Error is: %s\n", name, simdjson::errorMsg(parseRes).data());
          everythingfine = false;
-        }
-      } else if (startsWith("fail", name)) {
-        if (isok) {
+      } else if (startsWith("fail", name) && parseRes == 0) {
          isfileasexpected[i] = false;
          printf("warning: file %s should fail but it passes.\n", name);
          everythingfine = false;
-        }
      }
      free(fullpath);
    }
@@ -0,0 +1,21 @@
+#include <iostream>
+#include "../singleheader/simdjson.h"
+
+int main() {
+  const char * filename = JSON_TEST_PATH; 
+  std::string_view p = get_corpus(filename);
+  ParsedJson pj = build_parsed_json(p); // do the parsing
+  if( ! pj.isValid() ) {
+    return EXIT_FAILURE;
+  }
+  if( ! pj.allocateCapacity(p.size()) ) {
+    return EXIT_FAILURE;
+  }
+  const int res = json_parse(p, pj);
+  if (res) {
+    std::cerr << simdjson::errorMsg(res) << std::endl;
+    return EXIT_FAILURE;
+  }
+  aligned_free((void*)p.data());
+  return EXIT_SUCCESS;
+}
@@ -87,9 +87,9 @@ int main(int argc, char *argv[]) {
    std::cerr << "failed to allocate memory" << std::endl;
    return EXIT_FAILURE;
  }
-  bool is_ok = json_parse(p, pj); // do the parsing, return false on error
+  int res = json_parse(p, pj); // do the parsing, return false on error
  aligned_free((void *)p.data());
-  if (!is_ok) {
+  if (res) {
    std::cerr << " Parsing failed. " << std::endl;
    return EXIT_FAILURE;
  }
@@ -101,7 +101,7 @@ int main(int argc, char *argv[]) {
    }
    compute_dump(pjh);
  } else {
-    is_ok = rawdump ? pj.dump_raw_tape(std::cout) : pj.printjson(std::cout);
+    const bool is_ok = rawdump ? pj.dump_raw_tape(std::cout) : pj.printjson(std::cout);
    if (!is_ok) {
      std::cerr << " Could not print out parsed result. " << std::endl;
      return EXIT_FAILURE;
@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+########################################################################
+# Generates a new release.
+########################################################################
+import sys
+import re
+import subprocess
+import io
+import os
+
+
+def extractnumbers(s):
+    return tuple(map(int,re.findall("(\d+)\.(\d+)\.(\d+)",str(s))[0]))
+
+def toversionstring(major, minor, rev):
+    return str(major)+"."+str(minor)+"."+str(rev)
+
+def topaddedversionstring(major, minor, rev):
+    return str(major)+str(minor).zfill(3)+str(rev).zfill(3)
+
+pipe = subprocess.Popen(["git", "rev-parse", "--abbrev-ref", "HEAD"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+branchresult = pipe.communicate()[0].decode().strip()
+
+if(branchresult != "master"):
+    print("release on master, you are on '"+branchresult+"'")
+    sys.exit(-1)
+
+
+ret = subprocess.call(["git", "remote", "update"])
+
+if(ret != 0):
+    sys.exit(ret)
+
+
+
+pipe = subprocess.Popen(["git", "log", "HEAD..", "--oneline"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+uptodateresult = pipe.communicate()[0].decode().strip()
+
+if(len(uptodateresult) != 0):
+    print(uptodateresult)
+    sys.exit(-1)
+
+pipe = subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+maindir = pipe.communicate()[0].decode().strip()
+
+
+print("repository: "+maindir)
+
+pipe = subprocess.Popen(["git", "describe", "--abbrev=0", "--tags"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+versionresult = pipe.communicate()[0].decode().strip()
+
+print("last version: "+versionresult )
+try:
+  currentv = extractnumbers(versionresult)
+except:
+  currentv = [0,0,0]
+if(len(sys.argv) != 2):
+    nextv = (currentv[0],currentv[1], currentv[2]+1)
+    print ("please specify version number, e.g. "+toversionstring(*nextv))
+    sys.exit(-1)
+try:
+    newversion = extractnumbers(sys.argv[1])
+except:
+    print("can't parse version number "+sys.argv[1])
+    sys.exit(-1)
+
+print("checking that new version is valid")
+
+if(newversion[0] !=  currentv[0]):
+    assert newversion[0] ==  currentv[0] + 1
+    assert newversion[1] == 0
+    assert newversion[2] == 0
+elif (newversion[1] !=  currentv[1]):
+    assert newversion[1] ==  currentv[1] + 1
+    assert newversion[2] == 0
+else :
+    assert newversion[2] ==  currentv[2] + 1
+
+
+
+versionfilerel = os.sep + "include" + os.sep + "simdjson" + os.sep + "simdjson_version.h"
+versionfile = maindir + versionfilerel
+
+with open(versionfile, 'w') as file:
+    file.write("// "+versionfilerel+" automatically generated by release.py, do not change by hand \n")
+    file.write("#ifndef SIMDJSON_INCLUDE_SIMDJSON_VERSION \n")
+    file.write("#define SIMDJSON_INCLUDE_SIMDJSON_VERSION \n")
+    file.write("#define SIMDJSON_VERSION "+toversionstring(*newversion)+" \n")
+    file.write("enum { \n")
+    file.write("    SIMDJSON_VERSION_MAJOR = "+str(newversion[0])+",  \n")
+    file.write("    SIMDJSON_VERSION_MINOR = "+str(newversion[1])+",  \n")
+    file.write("    SIMDJSON_VERSION_REVISION = "+str(newversion[2])+"  \n")
+    file.write("}; \n")
+    file.write("#endif // SIMDJSON_INCLUDE_SIMDJSON_VERSION \n")
+
+
+print(versionfile + " modified")
+
+import fileinput
+import re
+
+newmajorversionstring = str(newversion[0])
+mewminorversionstring = str(newversion[1])
+newrevversionstring = str(newversion[2])
+newversionstring = str(newversion[0]) + "." + str(newversion[1]) + "." + str(newversion[2])
+cmakefile = maindir + os.sep + "CMakeLists.txt"
+for line in fileinput.input(cmakefile, inplace=1, backup='.bak'):
+    line = re.sub('SIMDJSON_LIB_VERSION "\d+\.\d+\.\d+','SIMDJSON_LIB_VERSION "'+newversionstring, line.rstrip())
+    line = re.sub('SIMDJSON_LIB_SOVERSION "\d+','SIMDJSON_LIB_SOVERSION "'+newmajorversionstring, line)
+    line = re.sub('set\(PROJECT_VERSION_MAJOR \d+','set(PROJECT_VERSION_MAJOR '+newmajorversionstring, line)
+    line = re.sub('set\(PROJECT_VERSION_MINOR \d+','set(PROJECT_VERSION_MINOR '+mewminorversionstring, line)
+    line = re.sub('set\(PROJECT_VERSION_PATCH \d+','set(PROJECT_VERSION_PATCH '+newrevversionstring, line)
+    print(line)
+
+
+print("modified "+cmakefile+", a backup was made")
+
+scriptlocation = os.path.dirname(os.path.abspath(__file__))
+
+
+print("Please run the tests before issuing a release, do make test && make amalgamate \n")
+print("to issue release, enter \n git commit -a && git push  &&  git tag -a v"+toversionstring(*newversion)+" -m \"version "+toversionstring(*newversion)+"\" &&  git push --tags \n")
Author	SHA1	Message	Date
Daniel Lemire	d5a185b13e	new release.	2019-03-13 20:02:44 -04:00
Daniel Lemire	40ad3d6c92	Removing garbage from script.	2019-03-13 20:02:44 -04:00
Daniel Lemire	df8f792183	Store the string lengths on the string tape (#101 ) * Store string length in the string-tape item. * Files are now limited to 4GB. * Moving detection of unescaped chars to stage 1 to reduce the burden due to string parsing. Fixes https://github.com/lemire/simdjson/issues/114 Fixes https://github.com/lemire/simdjson/issues/87	2019-03-13 19:32:57 -04:00
Daniel Lemire	609e96b5d1	Fix for https://github.com/lemire/simdjson/issues/119	2019-03-13 11:01:31 -04:00
myd7349	bb95747d56	Add instructions regarding vcpkg (fix #116 ) (#120 )	2019-03-13 08:15:27 -04:00
Daniel Lemire	afd9d72954	Updating project logo.	2019-03-12 16:11:36 -04:00
Daniel Lemire	596d79cde2	Displaying logo	2019-03-12 15:13:25 -04:00
Daniel Lemire	d9cc78615f	Fix for https://github.com/lemire/simdjson/issues/117	2019-03-12 15:09:47 -04:00
Daniel Lemire	55e9b082cb	Fixing typo	2019-03-11 08:54:37 -04:00
Daniel Lemire	70f8fc1b42	Update README.md	2019-03-11 08:10:23 -04:00
Daniel Lemire	da970585ca	Elaborating regarding single-header version.	2019-03-10 10:44:37 -04:00
myd7349	d2fa086198	Fix C4146 build error on UWP with MSVC (#113 ) * Fix C4146 build error on UWP with MSVC * Regenerate single header version * Fix typo in parsedjson.h * Regenerate single header version	2019-03-09 08:46:06 -05:00
geofflangdale	5dc47ac0ea	Merge pull request #110 from NanXiao/patch-1 Fix build warning	2019-03-07 17:58:12 +11:00
geofflangdale	f8566871a0	Merge pull request #111 from NanXiao/patch-2 Fix typo	2019-03-07 17:57:22 +11:00
Nan Xiao	c1dea29501	Fix typo	2019-03-07 13:55:41 +08:00
Nan Xiao	8fc77a4365	Fix build warning	2019-03-07 13:49:27 +08:00
Tyler Kennedy	21eef55907	Changes to the behaviour of move_forward to make it suitable for iteration. (Closes #73 ) (#103 )	2019-03-06 12:13:55 -05:00
Daniel Lemire	d5a35c8fc2	adding appveyor badge.	2019-03-06 10:14:22 -05:00
Georgios Floros	d873ee9983	Reuse aligned_malloc (#108 )	2019-03-06 10:12:55 -05:00
Geoff Langdale	6628c365c9	Substantial refactor (and clang-format google stype) of stage1_find_marks.cpp	2019-03-06 11:09:50 +11:00
Georgios Floros	0ca170b130	Fix crash in singleheader test (#105 ) Use `aligned_free` instead of `free` to free memory allocated by aligned function.	2019-03-05 15:35:34 -05:00
geofflangdale	5040840578	Merge pull request #102 from JakeConnors376W/patch-1 Improve wording	2019-03-05 11:26:56 +11:00
JakeConnors376W	0213c3b4d6	Improve wording	2019-03-04 16:12:09 -08:00
myd7349	2851ea490c	Export CMake targets (#96 )	2019-03-04 16:07:06 -05:00
Daniel Lemire	6541682433	Adding versions to cmake	2019-03-04 15:40:52 -05:00
Daniel Lemire	693b45c561	Fix to release string regarding so versions.	2019-03-04 15:23:45 -05:00
M. Zhou	6102b310b2	CMake: SOVERSION should be 0 instead of 0.0.1 (#98 )	2019-03-04 08:15:47 -05:00
geofflangdale	ec2b664d8e	Merge pull request #97 from monomagentaeggroll/master Update README.md	2019-03-04 10:47:18 +11:00
monomagentaeggroll	cd82418ee7	Update README.md Fixes consistency in presentation of information and grammatical tenses.	2019-03-03 15:17:08 -08:00
Thomas Navennec	352dd5e7fa	Change parse_json return type from bool to int (#82 ) * Added simdjerr namespace * Updated jsonparser files * updated stage1 and stage2 * removed stage2 inline function * Added forgotten return statements * Updated tools and benchmarks * Corrected parenthesis * Removed extra = * Accidentally undid reinterpret_cast * Better comments, undid a header name fuckup * Added an errorMsg method, updated readme * Removed useless header from stage2 * Updated single-header file * added simdjerr.cpp contents to simdjson.cpp * Made single header version work * Updated singleheader test, fixed simdjson.cpp * Renamed simdjerr namespace and files to simdjson * Updating the amalgamation.	2019-03-02 17:18:45 -05:00
Daniel Lemire	10b6b0445e	disabling appveyor badge for now.	2019-03-02 08:28:46 -05:00
Thomas Navennec	b0c4302887	Very basic test for single header (#91 ) * Very basic test for single header * Changed the way the test is declared	2019-02-27 21:04:06 -05:00
Luiz Fernando Peres	db1702623c	Including Node.js bindings to simdjson list of projects (#88 )	2019-02-27 10:39:31 -05:00
greedengineer	0c8ee105b4	fix memory free (#86 )	2019-02-27 07:50:20 -05:00
Carmot	a22c20fab0	Removed innecesary check and objects release. (#79 )	2019-02-26 19:08:13 -05:00
Daniel Lemire	9ddc8d6ba6	Adding release script.	2019-02-26 17:08:00 -05:00
M. Zhou	3d48628e71	CMake: Add version and soversion to library target properties. (#76 )	2019-02-26 16:39:59 -05:00
Daniel Lemire	696e7175b7	Update .gitattributes	2019-02-26 13:51:39 -05:00
Daniel Lemire	a78547835e	Simplifying the gitattributes commented line	2019-02-26 13:38:45 -05:00