This is a version of the avx-512 kernel which uses ternary instructions and lower-level constructions. It is not faster.

Making AVX-512 work by default.

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 2.0.0
+    VERSION 1.0.2
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C
@@ -20,8 +20,8 @@ string(
 # ---- Options, variables ----
 
 # These version numbers are modified by tools/release.py
-set(SIMDJSON_LIB_VERSION "11.0.0" CACHE STRING "simdjson library version")
-set(SIMDJSON_LIB_SOVERSION "11" CACHE STRING "simdjson library soversion")
+set(SIMDJSON_LIB_VERSION "9.0.0" CACHE STRING "simdjson library version")
+set(SIMDJSON_LIB_SOVERSION "9" CACHE STRING "simdjson library soversion")
 
 option(SIMDJSON_ENABLE_THREADS "Link with thread support" ON)
 

CONTRIBUTORS

@@ -40,6 +40,6 @@ Pavel Pavlov
 Hao Chen
 Nicolas Boyer
 Kim Walisch and Jatin Bhateja (AVX-512 bitset decoder)
-Fangzheng Zhang and Weiqiang Wan (AVX-512 kernel)
+Eric Zhang (AVX-512 kernel)
 # if you have contributed to the project and your name does not
 # appear in this list, please let us know!

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "2.0.0"
+PROJECT_NUMBER         = "1.0.2"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

benchmark/amazon_cellphones/amazon_cellphones.h

@@ -4,12 +4,8 @@
 #include <map>
 #include <string>
 
-
 namespace amazon_cellphones {
 
-const bool UNTHREADED = false;
-const bool THREADED = true;
-
 using namespace json_benchmark;
 
 struct brand {
@@ -63,11 +59,10 @@ struct runner : public file_runner<I> {
     }
 };
 
-template<bool threaded>
 struct simdjson_dom;
 
 template<typename I> simdjson_really_inline static void amazon_cellphones(benchmark::State &state) {
-  run_json_benchmark<runner<I>, runner<simdjson_dom<UNTHREADED>>>(state);
+  run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
 }
 
 }   // namespace amazon_cellphones

benchmark/amazon_cellphones/simdjson_dom.h

@@ -8,16 +8,12 @@ namespace amazon_cellphones {
 
 using namespace simdjson;
 
-template<bool threaded>
 struct simdjson_dom {
   using StringType = std::string;
 
   dom::parser parser{};
 
   bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
-#ifdef SIMDJSON_THREADS_ENABLED
-    parser.threaded = threaded;
-#endif
     auto stream = parser.parse_many(json);
     auto i = stream.begin();
     ++i;  // Skip first line
@@ -41,10 +37,7 @@ struct simdjson_dom {
 
 };
 
-BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_dom<UNTHREADED>)->UseManualTime();
-#ifdef SIMDJSON_THREADS_ENABLED
-BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_dom<THREADED>)->UseManualTime();
-#endif
+BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_dom)->UseManualTime();
 
 } // namespace amazon_cellphones
 

benchmark/amazon_cellphones/simdjson_ondemand.h

@@ -8,16 +8,12 @@ namespace amazon_cellphones {
 
 using namespace simdjson;
 
-template<bool threaded>
 struct simdjson_ondemand {
   using StringType = std::string;
 
   ondemand::parser parser{};
 
   bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
-#ifdef SIMDJSON_THREADS_ENABLED
-    parser.threaded = threaded;
-#endif
     ondemand::document_stream stream = parser.iterate_many(json);
     ondemand::document_stream::iterator i = stream.begin();
     ++i;  // Skip first line
@@ -62,10 +58,7 @@ struct simdjson_ondemand {
 
 };
 
-BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_ondemand<UNTHREADED>)->UseManualTime();
-#ifdef SIMDJSON_THREADS_ENABLED
-BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_ondemand<THREADED>)->UseManualTime();
-#endif
+BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_ondemand)->UseManualTime();
 
 } // namespace amazon_cellphones
 

benchmark/large_amazon_cellphones/large_amazon_cellphones.h

@@ -6,9 +6,6 @@
 
 namespace large_amazon_cellphones {
 
-const bool UNTHREADED = false;
-const bool THREADED = true;
-
 static const simdjson::padded_string &get_built_json();
 
 using namespace json_benchmark;
@@ -84,11 +81,11 @@ static const simdjson::padded_string &get_built_json() {
   return json;
 }
 
-template<bool threaded>
+
 struct simdjson_dom;
 
 template<typename I> simdjson_really_inline static void large_amazon_cellphones(benchmark::State &state) {
-  run_json_benchmark<runner<I>, runner<simdjson_dom<UNTHREADED>>>(state);
+  run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
 }
 
 }   // namespace large_amazon_cellphones

benchmark/large_amazon_cellphones/simdjson_dom.h

@@ -9,16 +9,12 @@ namespace large_amazon_cellphones {
 
 using namespace simdjson;
 
-template<bool threaded>
 struct simdjson_dom {
   using StringType = std::string;
 
   dom::parser parser{};
 
   bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
-#ifdef SIMDJSON_THREADS_ENABLED
-    parser.threaded = threaded;
-#endif
     auto stream = parser.parse_many(json);
     auto i = stream.begin();
     ++i;  // Skip first line
@@ -42,10 +38,7 @@ struct simdjson_dom {
 
 };
 
-BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_dom<UNTHREADED>)->UseManualTime();
-#ifdef SIMDJSON_THREADS_ENABLED
-BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_dom<THREADED>)->UseManualTime();
-#endif
+BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_dom)->UseManualTime();
 
 } // namespace large_amazon_cellphones
 

benchmark/large_amazon_cellphones/simdjson_ondemand.h

@@ -8,16 +8,12 @@ namespace large_amazon_cellphones {
 
 using namespace simdjson;
 
-template<bool threaded>
 struct simdjson_ondemand {
   using StringType = std::string;
 
   ondemand::parser parser{};
 
   bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
-#ifdef SIMDJSON_THREADS_ENABLED
-    parser.threaded = threaded;
-#endif
     ondemand::document_stream stream = parser.iterate_many(json);
     ondemand::document_stream::iterator i = stream.begin();
     ++i;  // Skip first line
@@ -62,10 +58,7 @@ struct simdjson_ondemand {
 
 };
 
-BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_ondemand<UNTHREADED>)->UseManualTime();
-#ifdef SIMDJSON_THREADS_ENABLED
-BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_ondemand<THREADED>)->UseManualTime();
-#endif
+BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_ondemand)->UseManualTime();
 
 } // namespace amazon_cellphones
 

doc/basics.md

@@ -1401,6 +1401,10 @@ You must check the type before accessing the value: it is an error to call `get_
 
 The `get_number()` function is designed with performance in mind. When calling `get_number()`, you scan the number string only once, determining efficiently the type and storing it in an efficient manner.
 
+If you only need to compute `v.get_number().get_number_type()` on
+a `document` or `value` instance, you should call directly the faster method
+`v.get_number_type()` which does not generate an
+intermediate `number` instance.
 
 Consider the following example:
 ```C++
@@ -1413,7 +1417,9 @@ Consider the following example:
       std::cout << "negative: " << val.is_negative() << " ";
       std::cout << "is_integer: " << val.is_integer() << " ";
       ondemand::number num = val.get_number();
-      ondemand::number_type t = num.get_number_type();
+      // direct computation without materializing the number:
+      ondemand::number_type dt = val.get_number_type();
+      if(t != dt) { throw std::runtime_error("bug"); }
       switch(t) {
         case ondemand::number_type::signed_integer:
           std::cout  << "integer: " << int64_t(num) << " ";

include/simdjson/generic/numberparsing.h

@@ -1067,11 +1067,6 @@ simdjson_unused simdjson_really_inline simdjson_result<ondemand::number_type> ge
   while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
   if ( p == src ) { return NUMBER_ERROR; }
   if (jsoncharutils::is_structural_or_whitespace(*p)) {
-    // We have an integer.
-    // If the number is negative and valid, it must be a signed integer.
-    if(negative) { return ondemand::number_type::signed_integer; }
-    // We want values larger or equal to 9223372036854775808 to be unsigned
-    // integers, and the other values to be signed integers.
     int digit_count = int(p - src);
     if(digit_count >= 19) {
       const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
@@ -1081,7 +1076,6 @@ simdjson_unused simdjson_really_inline simdjson_result<ondemand::number_type> ge
     }
     return ondemand::number_type::signed_integer;
   }
-  // Hopefully, we have 'e' or 'E' or '.'.
   return ondemand::number_type::floating_point_number;
 }
 

include/simdjson/generic/ondemand/document-inl.h

@@ -24,10 +24,6 @@ inline simdjson_result<const char *> document::current_location() noexcept {
   return iter.current_location();
 }
 
-inline int32_t document::current_depth() const noexcept {
-  return iter.depth();
-}
-
 inline bool document::is_alive() noexcept {
   return iter.is_alive();
 }
@@ -475,11 +471,6 @@ simdjson_really_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IM
   return first.current_location();
 }
 
-simdjson_really_inline int32_t simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::current_depth() const noexcept {
-  if (error()) { return error(); }
-  return first.current_depth();
-}
-
 simdjson_really_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::raw_json_token() noexcept {
   if (error()) { return error(); }
   return first.raw_json_token();
@@ -537,7 +528,6 @@ simdjson_really_inline simdjson_result<value> document_reference::find_field_uno
 simdjson_really_inline simdjson_result<json_type> document_reference::type() noexcept { return doc->type(); }
 simdjson_really_inline simdjson_result<bool> document_reference::is_scalar() noexcept { return doc->is_scalar(); }
 simdjson_really_inline simdjson_result<const char *> document_reference::current_location() noexcept { return doc->current_location(); }
-simdjson_really_inline int32_t document_reference::current_depth() const noexcept { return doc->current_depth(); }
 simdjson_really_inline bool document_reference::is_negative() noexcept { return doc->is_negative(); }
 simdjson_really_inline simdjson_result<bool> document_reference::is_integer() noexcept { return doc->is_integer(); }
 simdjson_really_inline simdjson_result<number_type> document_reference::get_number_type() noexcept { return doc->get_number_type(); }

include/simdjson/generic/ondemand/document.h

@@ -399,14 +399,7 @@ public:
    */
   simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
   /**
-   * Determine the number type (integer or floating-point number) as quickly
-   * as possible. This function does not fully validate the input. It is
-   * useful when you only need to classify the numbers, without parsing them.
-   *
-   * If you are planning to retrieve the value or you need full validation,
-   * consider using the get_number() method instead: it will fully parse
-   * and validate the input, and give you access to the type:
-   * get_number().get_number_type().
+   * Determine the number type (integer or floating-point number).
    *
    * get_number_type() is number_type::unsigned_integer if we have
    * an integer greater or equal to 9223372036854775808
@@ -414,7 +407,8 @@ public:
    * integer that is less than 9223372036854775808
    * Otherwise, get_number_type() has value number_type::floating_point_number
    *
-   * This function requires processing the number string, but it is expected
+   * This function req
+   * uires processing the number string, but it is expected
    * to be faster than get_number().get_number_type() because it is does not
    * parse the number value.
    *
@@ -495,17 +489,6 @@ public:
    */
   inline simdjson_result<const char *> current_location() noexcept;
 
-  /**
-   * Returns the current depth in the document if in bounds.
-   *
-   * E.g.,
-   *  0 = finished with document
-   *  1 = document root value (could be [ or {, not yet known)
-   *  2 = , or } inside root array/object
-   *  3 = key or value inside root array/object.
-   */
-  simdjson_really_inline int32_t current_depth() const noexcept;
-
   /**
    * Get the value associated with the given JSON pointer.  We use the RFC 6901
    * https://tools.ietf.org/html/rfc6901 standard.
@@ -626,7 +609,6 @@ public:
   simdjson_really_inline simdjson_result<bool> is_scalar() noexcept;
 
   simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
-  simdjson_really_inline int32_t current_depth() const noexcept;
   simdjson_really_inline bool is_negative() noexcept;
   simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
   simdjson_really_inline simdjson_result<number_type> get_number_type() noexcept;
@@ -693,7 +675,6 @@ public:
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_type> type() noexcept;
   simdjson_really_inline simdjson_result<bool> is_scalar() noexcept;
   simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
-  simdjson_really_inline int32_t current_depth() const noexcept;
   simdjson_really_inline bool is_negative() noexcept;
   simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> get_number_type() noexcept;
@@ -754,7 +735,6 @@ public:
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_type> type() noexcept;
   simdjson_really_inline simdjson_result<bool> is_scalar() noexcept;
   simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
-  simdjson_really_inline int32_t current_depth() const noexcept;
   simdjson_really_inline bool is_negative() noexcept;
   simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> get_number_type() noexcept;

include/simdjson/generic/ondemand/json_iterator.h

@@ -154,7 +154,7 @@ public:
    * This is not null-terminated; it is a view into the JSON.
    *
    * You may be pointing outside of the input buffer: it is not generally
-   * safe to dereference this pointer.
+   * safe to derefence this pointer.
    */
   simdjson_really_inline const uint8_t *unsafe_pointer() const noexcept;
   /**

include/simdjson/generic/ondemand/logger-inl.h

@@ -85,7 +85,7 @@ inline void log_headers() noexcept {
       printf("# skip says 'this is a structural or value I am skipping'\n");
       printf("# +/-skip says 'this is a start/end array or object I am skipping'\n");
       printf("#\n");
-      printf("# The indentation of the terms (array, string,...) indicates the depth,\n");
+      printf("# The identation of the terms (array, string,...) indicates the depth,\n");
       printf("# in addition to the depth being displayed.\n");
       printf("#\n");
       printf("# Every token in the document has a single depth determined by the tokens before it,\n");

include/simdjson/generic/ondemand/object.h

@@ -118,7 +118,7 @@ public:
    * instance: there is no rewind and no invalidation.
    *
    * You may call at_pointer more than once on an object, but each time the pointer is advanced
-   * to be within the value matched by the key indicated by the JSON pointer query. Thus any preceding
+   * to be within the value matched by the key indicated by the JSON pointer query. Thus any preceeding
    * key (as well as the current key) can no longer be used with following JSON pointer calls.
    *
    * Also note that at_pointer() relies on find_field() which implies that we do not unescape keys when matching.
@@ -168,7 +168,7 @@ public:
   simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
   /**
    * Consumes the object and returns a string_view instance corresponding to the
-   * object as represented in JSON. It points inside the original byte array containing
+   * object as represented in JSON. It points inside the original byte array containg
    * the JSON document.
    */
   simdjson_really_inline simdjson_result<std::string_view> raw_json() noexcept;

include/simdjson/generic/ondemand/value-inl.h

@@ -181,10 +181,6 @@ simdjson_really_inline simdjson_result<const char *> value::current_location() n
   return iter.json_iter().current_location();
 }
 
-simdjson_really_inline int32_t value::current_depth() const noexcept{
-  return iter.json_iter().depth();
-}
-
 simdjson_really_inline simdjson_result<value> value::at_pointer(std::string_view json_pointer) noexcept {
   json_type t;
   SIMDJSON_TRY(type().get(t));
@@ -404,11 +400,6 @@ simdjson_really_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IM
   return first.current_location();
 }
 
-simdjson_really_inline int32_t simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::current_depth() const noexcept {
-  if (error()) { return error(); }
-  return first.current_depth();
-}
-
 simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::at_pointer(std::string_view json_pointer) noexcept {
   if (error()) { return error(); }
   return first.at_pointer(json_pointer);

include/simdjson/generic/ondemand/value.h

@@ -369,14 +369,7 @@ public:
    */
   simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
   /**
-   * Determine the number type (integer or floating-point number) as quickly
-   * as possible. This function does not fully validate the input. It is
-   * useful when you only need to classify the numbers, without parsing them.
-   *
-   * If you are planning to retrieve the value or you need full validation,
-   * consider using the get_number() method instead: it will fully parse
-   * and validate the input, and give you access to the type:
-   * get_number().get_number_type().
+   * Determine the number type (integer or floating-point number).
    *
    * get_number_type() is number_type::unsigned_integer if we have
    * an integer greater or equal to 9223372036854775808
@@ -455,17 +448,6 @@ public:
    */
   simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
 
-  /**
-   * Returns the current depth in the document if in bounds.
-   *
-   * E.g.,
-   *  0 = finished with document
-   *  1 = document root value (could be [ or {, not yet known)
-   *  2 = , or } inside root array/object
-   *  3 = key or value inside root array/object.
-   */
-  simdjson_really_inline int32_t current_depth() const noexcept;
-
   /**
    * Get the value associated with the given JSON pointer.  We use the RFC 6901
    * https://tools.ietf.org/html/rfc6901 standard.
@@ -497,7 +479,7 @@ public:
    * to call at_pointer on the same array.
    *
    * You may call at_pointer more than once on an object, but each time the pointer is advanced
-   * to be within the value matched by the key indicated by the JSON pointer query. Thus any preceding
+   * to be within the value matched by the key indicated by the JSON pointer query. Thus any preceeding
    * key (as well as the current key) can no longer be used with following JSON pointer calls.
    *
    * Also note that at_pointer() relies on find_field() which implies that we do not unescape keys when matching
@@ -668,8 +650,7 @@ public:
 
   /** @copydoc simdjson_really_inline simdjson_result<const char *> current_location() noexcept */
   simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
-  /** @copydoc simdjson_really_inline int32_t current_depth() const noexcept */
-  simdjson_really_inline int32_t current_depth() const noexcept;
+
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at_pointer(std::string_view json_pointer) noexcept;
 };
 

include/simdjson/generic/ondemand/value_iterator.h

@@ -76,7 +76,7 @@ public:
   /**
    * Get the depth of this value.
    */
-  simdjson_really_inline int32_t depth() const noexcept;
+  simdjson_really_inline depth_t depth() const noexcept;
 
   /**
    * Get the JSON type of this value.

include/simdjson/icelake/simd.h

@@ -328,6 +328,8 @@ namespace simd {
       const simd8<T> mask = simd8<T>::splat(m);
       return this->chunks[0] <= mask;
     }
+    simdjson_really_inline operator __m512i() const { return __m512i(this->chunks[0]); }
+
   }; // struct simd8x64<T>
 
 } // namespace simd

include/simdjson/simdjson_version.h

@@ -4,14 +4,14 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION 2.0.0
+#define SIMDJSON_VERSION 1.0.2
 
 namespace simdjson {
 enum {
   /**
    * The major version (MAJOR.minor.revision) of simdjson being used.
    */
-  SIMDJSON_VERSION_MAJOR = 2,
+  SIMDJSON_VERSION_MAJOR = 1,
   /**
    * The minor version (major.MINOR.revision) of simdjson being used.
    */
@@ -19,7 +19,7 @@ enum {
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 0
+  SIMDJSON_VERSION_REVISION = 2
 };
 } // namespace simdjson
 

singleheader/amalgamate.py

@@ -9,7 +9,6 @@ import subprocess
 import os
 import re
 import shutil
-import datetime
 
 if sys.version_info < (3, 0):
     sys.stdout.write("Sorry, requires Python 3.x or better\n")
@@ -122,12 +121,8 @@ def dofile(fid, prepath, filename):
 # does not change with locale and timezone at time of generation.
 # Forcing it to be UTC is difficult, because it needs to be portable
 # between gnu date and busybox date.
-try:
-    timestamp = subprocess.run(['git', 'show', '-s', '--format=%ci', 'HEAD'],
+timestamp = subprocess.run(['git', 'show', '-s', '--format=%ci', 'HEAD'],
                            stdout=subprocess.PIPE).stdout.decode('utf-8').strip()
-except:
-    print("git not found, timestamp based on current time")
-    timestamp = str(datetime.datetime.now())
 print(f"timestamp is {timestamp}")
 
 os.makedirs(AMALGAMATE_OUTPUT_PATH, exist_ok=True)

src/icelake/dom_parser_implementation.cpp

@@ -102,7 +102,7 @@ simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t>
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#include "generic/stage1/utf8_lookup4_algorithm.h"
+#include "icelake/utf8_lookup4_algorithm.h"
 // defining SIMDJSON_CUSTOM_BIT_INDEXER allows us to provide our own bit_indexer::write
 #define SIMDJSON_CUSTOM_BIT_INDEXER
 #include "generic/stage1/json_structural_indexer.h"

src/icelake/utf8_lookup4_algorithm.h

@@ -0,0 +1,141 @@
+namespace simdjson {
+namespace SIMDJSON_IMPLEMENTATION {
+namespace {
+namespace utf8_validation {
+
+simdjson_really_inline __m512i check_special_cases(__m512i input, const __m512i prev1) {
+  __m512i mask1 = _mm512_setr_epi64(
+        0x0202020202020202,
+        0x4915012180808080,
+        0x0202020202020202,
+        0x4915012180808080,
+        0x0202020202020202,
+        0x4915012180808080,
+        0x0202020202020202,
+        0x4915012180808080);
+
+    const __m512i v_0f = _mm512_set1_epi8(0x0f);
+    __m512i index1 = _mm512_and_si512(_mm512_srli_epi16(prev1, 4), v_0f);
+
+    __m512i byte_1_high = _mm512_shuffle_epi8(mask1, index1);
+    __m512i mask2 = _mm512_setr_epi64(
+        0xcbcbcb8b8383a3e7,
+        0xcbcbdbcbcbcbcbcb,
+        0xcbcbcb8b8383a3e7,
+        0xcbcbdbcbcbcbcbcb,
+        0xcbcbcb8b8383a3e7,
+        0xcbcbdbcbcbcbcbcb,
+        0xcbcbcb8b8383a3e7,
+        0xcbcbdbcbcbcbcbcb);
+     __m512i index2 = _mm512_and_si512(prev1, v_0f);
+
+    __m512i byte_1_low = _mm512_shuffle_epi8(mask2, index2);
+    __m512i mask3 = _mm512_setr_epi64(
+        0x101010101010101,
+        0x1010101babaaee6,
+        0x101010101010101,
+        0x1010101babaaee6,
+        0x101010101010101,
+        0x1010101babaaee6,
+        0x101010101010101,
+        0x1010101babaaee6
+    );
+    __m512i index3 = _mm512_and_si512(_mm512_srli_epi16(input, 4), v_0f);
+    __m512i byte_2_high = _mm512_shuffle_epi8(mask3, index3);
+    return _mm512_ternarylogic_epi64(byte_1_high, byte_1_low, byte_2_high, 128);
+  }
+
+  simdjson_really_inline __m512i check_multibyte_lengths(const __m512i prev2,
+      const __m512i prev3, const __m512i sc) {
+
+    __m512i is_third_byte  = _mm512_subs_epu8(prev2, _mm512_set1_epi8(0b11100000u-1)); // Only 111_____ will be > 0
+    __m512i is_fourth_byte  = _mm512_subs_epu8(prev3, _mm512_set1_epi8(0b11110000u-1)); // Only 1111____ will be > 0
+    __m512i is_third_or_fourth_byte = _mm512_or_si512(is_third_byte, is_fourth_byte);
+    const __m512i v_7f = _mm512_set1_epi8(char(0x7f));
+    is_third_or_fourth_byte = _mm512_adds_epu8(v_7f, is_third_or_fourth_byte);
+    // We want to compute (is_third_or_fourth_byte AND v80) XOR sc.
+    const __m512i v_80 = _mm512_set1_epi8(char(0x80));
+    return _mm512_ternarylogic_epi32(is_third_or_fourth_byte, v_80, sc, 0b1101010);
+    // We could also do it the long way:
+    //
+    //__m512i is_third_or_fourth_byte_mask = _mm512_and_si512(is_third_or_fourth_byte, v_80);
+    //return _mm512_xor_si512(is_third_or_fourth_byte_mask, sc);
+  }
+  //
+  // Return nonzero if there are incomplete multibyte characters at the end of the block:
+  // e.g. if there is a 4-byte character, but it's 3 bytes from the end.
+  //
+  simdjson_really_inline __m512i is_incomplete(const __m512i input) {
+    // If the previous input's last 3 bytes match this, they're too short (they ended at EOF):
+    // ... 1111____ 111_____ 11______
+    const __m512i max_value = _mm512_setr_epi64(
+        0xffffffffffffffff,
+        0xffffffffffffffff,
+        0xffffffffffffffff,
+        0xffffffffffffffff,
+        0xffffffffffffffff,
+        0xffffffffffffffff,
+        0xffffffffffffffff,
+        0xbfdfefffffffffff);
+    return _mm512_subs_epu8(input, max_value);
+  }
+
+  struct utf8_checker {
+    // If this is nonzero, there has been a UTF-8 error.
+    __m512i error{};
+
+    // The last input we received
+    __m512i prev_input_block{};
+    // Whether the last input we received was incomplete (used for ASCII fast path)
+    __m512i prev_incomplete{};
+
+    //
+    // Check whether the current bytes are valid UTF-8.
+    //
+    simdjson_really_inline void check_utf8_bytes(const __m512i input, const __m512i prev_input) {
+      // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes
+      // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers)
+      const __m512i movemask = _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6);
+      const __m512i rotated = _mm512_permutex2var_epi64(prev_input, movemask, input);
+      __m512i prev1 = _mm512_alignr_epi8(input, rotated, 16-1);
+      __m512i prev2 = _mm512_alignr_epi8(input, rotated, 16-2);
+      __m512i prev3 = _mm512_alignr_epi8(input, rotated, 16-3);
+      __m512i sc = check_special_cases(input, prev1);
+      this->error = _mm512_or_si512(check_multibyte_lengths(prev2, prev3, sc), this->error);
+    }
+
+    // The only problem that can happen at EOF is that a multibyte character is too short
+    // or a byte value too large in the last bytes: check_special_cases only checks for bytes
+    // too large in the first of two bytes.
+    simdjson_really_inline void check_eof() {
+      // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't
+      // possibly finish them.
+      this->error = _mm512_or_si512(this->error, this->prev_incomplete);
+    }
+
+    // returns true if ASCII.
+    simdjson_really_inline bool check_next_input(const __m512i input) {
+      const __m512i v_80 = _mm512_set1_epi8(char(0x80));
+      const __mmask64 ascii = _mm512_test_epi8_mask(input, v_80);
+      if(ascii == 0) {
+        this->error = _mm512_or_si512(this->error, this->prev_incomplete);
+        return true;
+      } else {
+        this->check_utf8_bytes(input, this->prev_input_block);
+        this->prev_incomplete = is_incomplete(input);
+        this->prev_input_block = input;
+        return false;
+      }
+    }
+    // do not forget to call check_eof!
+    simdjson_really_inline error_code errors() {
+      return (_mm512_test_epi8_mask(this->error, this->error) != 0) ? error_code::UTF8_ERROR : error_code::SUCCESS;
+    }
+  }; // struct utf8_checker
+} // namespace utf8_validation
+
+using utf8_validation::utf8_checker;
+
+} // unnamed namespace
+} // namespace SIMDJSON_IMPLEMENTATION
+} // namespace simdjson

tests/ondemand/ondemand_array_error_tests.cpp

@@ -16,7 +16,7 @@ namespace array_error_tests {
     size_t count = 0;
     for (auto elem : std::forward<T>(array)) {
       std::cout << "-"; std::cout.flush();
-      V actual{};
+      V actual;
       auto actual_error = elem.get(actual);
       if (count >= N) {
         if (count >= (N+N2)) {

tests/ondemand/ondemand_number_tests.cpp

@@ -197,18 +197,16 @@ namespace number_tests {
   bool get_number_tests() {
     TEST_START();
     ondemand::parser parser;
-    padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999,-9223372036854775807,-9223372036854775808])"_padded;
+    padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999])"_padded;
     ondemand::number_type expectedtypes[] = {ondemand::number_type::floating_point_number,
                                  ondemand::number_type::signed_integer,
                                  ondemand::number_type::signed_integer,
                                  ondemand::number_type::floating_point_number,
                                  ondemand::number_type::signed_integer,
-                                 ondemand::number_type::unsigned_integer,
-                                 ondemand::number_type::signed_integer,
-                                 ondemand::number_type::signed_integer
+                                 ondemand::number_type::unsigned_integer
                                  };
-    bool is_negative[] = {false, false, false, false, true, false, true, true};
-    bool is_integer[] = {false, true, true, false, true, true, true, true};
+    bool is_negative[] = {false, false, false, false, true, false};
+    bool is_integer[] = {false, true, true, false, true, true};
 
     ondemand::document doc;
     ASSERT_SUCCESS(parser.iterate(docdata).get(doc));

tests/ondemand/ondemand_readme_examples.cpp

@@ -109,85 +109,6 @@ bool basics_treewalk() {
   return true;
 }
 
-void print_depth_space(ondemand::value element) {
-  for(auto i = 0; i < element.current_depth(); i++) {
-    cout << " ";
-  }
-}
-
-void recursive_print_json_breakline(ondemand::value element) {
-    bool add_comma;
-    switch (element.type()) {
-    case ondemand::json_type::array:
-      cout << endl;
-      print_depth_space(element);
-      cout << "[";
-      add_comma = false;
-      for (auto child : element.get_array()) {
-        if (add_comma) {
-          print_depth_space(element);
-          cout << ",";
-        }
-        // We need the call to value() to get
-        // an ondemand::value type.
-        recursive_print_json_breakline(child.value());
-        add_comma = true;
-      }
-      cout << "]";
-      break;
-    case ondemand::json_type::object:
-      cout << endl;
-      print_depth_space(element);
-      cout << "{";
-      add_comma = false;
-      for (auto field : element.get_object()) {
-        if (add_comma) {
-          print_depth_space(element);
-          cout << ",";
-        }
-        // key() returns the key as it appears in the raw
-        // JSON document, if we want the unescaped key,
-        // we should do field.unescaped_key().
-        cout << "\"" << field.key() << "\": ";
-        recursive_print_json_breakline(field.value());
-        add_comma = true;
-      }
-      cout << "}\n";
-      break;
-    case ondemand::json_type::number:
-      // assume it fits in a double
-      cout << element.get_double();
-      break;
-    case ondemand::json_type::string:
-      // get_string() would return escaped string, but
-      // we are happy with unescaped string.
-      cout << "\"" << element.get_raw_json_string() << "\"";
-      break;
-    case ondemand::json_type::boolean:
-      cout << element.get_bool();
-      break;
-    case ondemand::json_type::null:
-      cout << "null";
-      break;
-    }
-}
-
-bool basics_treewalk_breakline() {
-  padded_string json[3] = {R"( [
-    { "make": "Toyota", "model": "Camry",  "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
-    { "make": "Kia",    "model": "Soul",   "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
-    { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
-  ] )"_padded, R"( {"key":"value"} )"_padded, "[12,3]"_padded};
-  ondemand::parser parser;
-  for(size_t i = 0 ; i < 3; i++) {
-    ondemand::document doc = parser.iterate(json[i]);
-    ondemand::value val = doc;
-    recursive_print_json_breakline(val);
-    std::cout << std::endl;
-  }
-  return true;
-}
-
 bool basics_1() {
   TEST_START();
 
@@ -752,7 +673,7 @@ bool stream_capacity_example() {
   if( error ) { /* handle the error */ }
   for (auto doc: stream) {
     if(counter < 6) {
-      int64_t val{};
+      int64_t val;
       error = doc.at_pointer("/4").get(val);
       if( error ) { /* handle the error */ }
       std::cout << "5 = " << val << std::endl;
@@ -961,7 +882,6 @@ bool current_location_no_error() {
 int main() {
 #if SIMDJSON_EXCEPTIONS
   basics_treewalk();
-  basics_treewalk_breakline();
 #endif
   if (
     true