Disabling again for GCC 7

Patch for GCC 8.

.github/workflows/alpine.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/fix-trailing-whitespace.yml

@@ -6,7 +6,7 @@ jobs:
   whitespace:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Remove whitespace and check the diff
         run: |
           set -eu

.github/workflows/fuzzers.yml

@@ -37,14 +37,14 @@ jobs:
         chmod +x llvm.sh
         sudo ./llvm.sh $CLANGVERSION
 
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
 
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       with:
         path: dependencies/.cache
         key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
 
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
       id: cache-corpus
       with:
         path: out/

.github/workflows/macos-11.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/macos.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/msys2-clang.yml

@@ -24,8 +24,8 @@ jobs:
       CMAKE_GENERATOR: Ninja
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/msys2.yml

@@ -26,8 +26,8 @@ jobs:
       CMAKE_GENERATOR: Ninja
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu18-checkperf.yml

@@ -15,8 +15,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu18-oldclang.yml

@@ -13,8 +13,8 @@ jobs:
       CC: clang-7
       CXX: clang++-7
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu18-threadsani.yml

@@ -10,8 +10,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu18.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20-checkperf.yml

@@ -15,8 +15,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20-gcc8.yml

@@ -0,0 +1,38 @@
+name: Ubuntu 20.04 CI (GCC 8)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-20.04
+    env:
+      CXX: g++-8
+      CC: gcc-8
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Install GCC 8
+        run: sudo apt-get install -y g++-8
+      - name: Use cmake
+        run: |
+          mkdir builddebug &&
+          cd builddebug &&
+          cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cd .. &&
+          mkdir build &&
+          cd build &&
+          cmake  -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly   &&
+          cmake --install . &&
+          echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
+          cd ../tests/installation_tests/find &&
+          mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. &&  cmake --build .

.github/workflows/ubuntu20-noexcept.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20-nothread.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20-sani.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20-threadsani.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu20.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/ubuntu22-clang13.yml

@@ -0,0 +1,25 @@
+name: Ubuntu 22.04 CI (CLANG 13)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Install clang++-13
+        run: sudo apt-get install -y clang++-13
+      - name: Use cmake
+        run: |
+          mkdir build &&
+          cd build &&
+          CXX=clang++-13 cmake -DSIMDJSON_DEVELOPER_MODE=ON ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly

.github/workflows/ubuntu22-gcc12.yml

@@ -0,0 +1,25 @@
+name: Ubuntu 22.04 CI (GCC 12)
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    if: >-
+      ! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
+      ! contains(toJSON(github.event.commits.*.message), '[skip github]')
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: dependencies/.cache
+          key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
+      - name: Install gcc12
+        run: sudo apt-get install -y g++-12
+      - name: Use cmake
+        run: |
+          mkdir build &&
+          cd build &&
+          CXX=g++-12 cmake -DSIMDJSON_DEVELOPER_MODE=ON ..  &&
+          cmake --build .   &&
+          ctest -j --output-on-failure -LE explicitonly

.github/workflows/ubuntu22.yml

@@ -9,8 +9,8 @@ jobs:
       ! contains(toJSON(github.event.commits.*.message), '[skip github]')
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
         with:
           path: dependencies/.cache
           key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

.github/workflows/vs17-arm-ci.yml

@@ -14,7 +14,7 @@ jobs:
            - {arch: ARM64}
      steps:
        - name: checkout
-         uses: actions/checkout@v2
+         uses: actions/checkout@v3
        - name: Use cmake
          run: |
            cmake -A ${{ matrix.arch }} -DCMAKE_CROSSCOMPILING=1 -DSIMDJSON_DEVELOPER_MODE=ON -D SIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_EXCEPTIONS=OFF -B build  &&

.github/workflows/vs17-ci.yml

@@ -19,7 +19,7 @@ jobs:
           - {gen: Visual Studio 17 2022, arch: x64, shared: OFF}
     steps:
     - name: checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Configure
       run: |
         cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build

.github/workflows/vs17-clang-ci.yml

@@ -16,7 +16,7 @@ jobs:
           - {gen: Visual Studio 17 2022, arch: x64}
     steps:
     - name: checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Configure
       run: |
         cmake -G "${{matrix.gen}}" -A ${{matrix.arch}}  -T ClangCL -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build

.github/workflows/vs17-noexcept-ci.yml

@@ -7,8 +7,8 @@ jobs:
     name: windows-vs17
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/cache@v2
+    - uses: actions/checkout@v3
+    - uses: actions/cache@v3
       with:
         path: dependencies/.cache
         key: ${{ hashFiles('dependencies/CMakeLists.txt') }}

CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.14)
 project(
     simdjson
     # The version number is modified by tools/release.py
-    VERSION 1.0.2
+    VERSION 2.0.4
     DESCRIPTION "Parsing gigabytes of JSON per second"
     HOMEPAGE_URL "https://simdjson.org/"
     LANGUAGES CXX C
@@ -20,8 +20,8 @@ string(
 # ---- Options, variables ----
 
 # These version numbers are modified by tools/release.py
-set(SIMDJSON_LIB_VERSION "9.0.0" CACHE STRING "simdjson library version")
-set(SIMDJSON_LIB_SOVERSION "9" CACHE STRING "simdjson library soversion")
+set(SIMDJSON_LIB_VERSION "11.0.0" CACHE STRING "simdjson library version")
+set(SIMDJSON_LIB_SOVERSION "11" CACHE STRING "simdjson library soversion")
 
 option(SIMDJSON_ENABLE_THREADS "Link with thread support" ON)
 

CONTRIBUTING.md

@@ -62,13 +62,13 @@ Pull Requests
 
 Pull requests are always invited. However, we ask that you follow these guidelines:
 
-- It is wiser to discuss your ideas first as part of an issue before you start coding. If you omit this step and code first, be prepare to have your code receive scrutiny and be dropped.
-- Users should provide a rationale for their changes. Does it improve performance? Does it add a feature? Does it improve maintainability? Does fix a bug? This must be explicitly stated as part of the pull request. Do not propose changes based on taste or intuition. We do not delegate programming to tools: that some tool suggested a code change is not reason enough to change the code.
+- It is wise to discuss your ideas first as part of an issue before you start coding. If you omit this step and code first, be prepared to have your code receive scrutiny and be dropped.
+- Users should provide a rationale for their changes. Does it improve performance? Does it add a feature? Does it improve maintainability? Does it fix a bug? This must be explicitly stated as part of the pull request. Do not propose changes based on taste or intuition. We do not delegate programming to tools: that some tool suggested a code change is not reason enough to change the code.
    1. When your code improves performance, please document the gains with a benchmark using hard numbers.
-   2. If your code fixes a bug, please be either fix a failing test, or propose a new test.
+   2. If your code fixes a bug, please either fix a failing test, or propose a new test.
    3. Other types of changes must be clearly motivated. We openly discourage changes with no identifiable benefits.
 - Changes should be focused and minimal. You should change as few lines of code as possible. Please do not reformat or touch files needlessly.
-- New features must be accompanied of new tests, in general.
+- New features must be accompanied by new tests, in general.
 - Your code should pass our continuous-integration tests. It is your responsibility to ensure that your proposal pass the tests. We do not merge pull requests that would break our build.
    - An exception to this would be changes to non-code files, such as documentation and assets, or trivial changes to code, such as comments, where it is encouraged to explicitly ask for skipping a CI run using the `[skip ci]` prefix in your Pull Request title **and** in the first line of the most recent commit in a push. Example for such a commit: `[skip ci] Fixed typo in power_of_ten's docs`
    This benefits the project in such a way that the CI pipeline is not burdened by running jobs on changes that don't change any behavior in the code, which reduces wait times for other Pull Requests that do change behavior and require testing.

CONTRIBUTORS

@@ -40,6 +40,6 @@ Pavel Pavlov
 Hao Chen
 Nicolas Boyer
 Kim Walisch and Jatin Bhateja (AVX-512 bitset decoder)
-Eric Zhang (AVX-512 kernel)
+Fangzheng Zhang and Weiqiang Wan (AVX-512 kernel)
 # if you have contributed to the project and your name does not
 # appear in this list, please let us know!

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "1.0.2"
+PROJECT_NUMBER         = "2.0.4"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

README.md

@@ -3,7 +3,7 @@
 [![Ubuntu 20.04 CI](https://github.com/simdjson/simdjson/workflows/Ubuntu%2020.04%20CI%20(GCC%209)/badge.svg)](https://simdjson.org/plots.html)
 ![VS16-CI](https://github.com/simdjson/simdjson/workflows/VS16-CI/badge.svg)
 ![MinGW64-CI](https://github.com/simdjson/simdjson/workflows/MinGW64-CI/badge.svg)
-[![][license img]][license]  [![Doxygen Documentation](https://img.shields.io/badge/docs-doxygen-green.svg)](https://simdjson.org/api/1.0.0/index.html)
+[![][license img]][license]  [![Doxygen Documentation](https://img.shields.io/badge/docs-doxygen-green.svg)](https://simdjson.org/api/2.0.0/index.html)
 
 simdjson : Parsing gigabytes of JSON per second
 ===============================================

benchmark/amazon_cellphones/amazon_cellphones.h

@@ -4,8 +4,12 @@
 #include <map>
 #include <string>
 
+
 namespace amazon_cellphones {
 
+const bool UNTHREADED = false;
+const bool THREADED = true;
+
 using namespace json_benchmark;
 
 struct brand {
@@ -59,10 +63,11 @@ struct runner : public file_runner<I> {
     }
 };
 
+template<bool threaded>
 struct simdjson_dom;
 
 template<typename I> simdjson_really_inline static void amazon_cellphones(benchmark::State &state) {
-  run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
+  run_json_benchmark<runner<I>, runner<simdjson_dom<UNTHREADED>>>(state);
 }
 
 }   // namespace amazon_cellphones

benchmark/amazon_cellphones/simdjson_dom.h

@@ -8,12 +8,16 @@ namespace amazon_cellphones {
 
 using namespace simdjson;
 
+template<bool threaded>
 struct simdjson_dom {
   using StringType = std::string;
 
   dom::parser parser{};
 
   bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
+#ifdef SIMDJSON_THREADS_ENABLED
+    parser.threaded = threaded;
+#endif
     auto stream = parser.parse_many(json);
     auto i = stream.begin();
     ++i;  // Skip first line
@@ -37,7 +41,10 @@ struct simdjson_dom {
 
 };
 
-BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_dom)->UseManualTime();
+BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_dom<UNTHREADED>)->UseManualTime();
+#ifdef SIMDJSON_THREADS_ENABLED
+BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_dom<THREADED>)->UseManualTime();
+#endif
 
 } // namespace amazon_cellphones
 

benchmark/amazon_cellphones/simdjson_ondemand.h

@@ -8,12 +8,16 @@ namespace amazon_cellphones {
 
 using namespace simdjson;
 
+template<bool threaded>
 struct simdjson_ondemand {
   using StringType = std::string;
 
   ondemand::parser parser{};
 
   bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
+#ifdef SIMDJSON_THREADS_ENABLED
+    parser.threaded = threaded;
+#endif
     ondemand::document_stream stream = parser.iterate_many(json);
     ondemand::document_stream::iterator i = stream.begin();
     ++i;  // Skip first line
@@ -58,7 +62,10 @@ struct simdjson_ondemand {
 
 };
 
-BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_ondemand)->UseManualTime();
+BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_ondemand<UNTHREADED>)->UseManualTime();
+#ifdef SIMDJSON_THREADS_ENABLED
+BENCHMARK_TEMPLATE(amazon_cellphones, simdjson_ondemand<THREADED>)->UseManualTime();
+#endif
 
 } // namespace amazon_cellphones
 

benchmark/benchfeatures.cpp

@@ -204,10 +204,13 @@ struct feature_benchmarker {
   }
   // Rate of 1-7-structural misses per 8-structural flip
   double struct1_7_miss_rate(BenchmarkStage stage) const {
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    return 1;
+#else
     if (!has_events()) { return 1; }
     return struct7_miss[stage].best.branch_misses() - struct7[stage].best.branch_misses() / double(struct7_miss.stats->blocks_with_1_structural_flipped);
+#endif
   }
-
   // Extra cost of an 8-15 structural block over a 1-7 structural block
   double struct8_15_cost(BenchmarkStage stage) const {
     return cost_per_block(stage, struct15, struct15.stats->blocks_with_8_structurals, struct7);
@@ -218,8 +221,12 @@ struct feature_benchmarker {
   }
   // Rate of 8-15-structural misses per 8-structural flip
   double struct8_15_miss_rate(BenchmarkStage stage) const {
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    return 1;
+#else
     if (!has_events()) { return 1; }
     return double(struct15_miss[stage].best.branch_misses() - struct15[stage].best.branch_misses()) / double(struct15_miss.stats->blocks_with_8_structurals_flipped);
+#endif
   }
 
   // Extra cost of a 16+-structural block over an 8-15 structural block (actual varies based on # of structurals!)
@@ -232,10 +239,15 @@ struct feature_benchmarker {
   }
   // Rate of 16-structural misses per 16-structural flip
   double struct16_miss_rate(BenchmarkStage stage) const {
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    return 1;
+#else
     if (!has_events()) { return 1; }
     return double(struct23_miss[stage].best.branch_misses() - struct23[stage].best.branch_misses()) / double(struct23_miss.stats->blocks_with_16_structurals_flipped);
+#endif
   }
 
+
   // Extra cost of having UTF-8 in a block
   double utf8_cost(BenchmarkStage stage) const {
     return cost_per_block(stage, utf8, utf8.stats->blocks_with_utf8, struct7_full);
@@ -246,10 +258,13 @@ struct feature_benchmarker {
   }
   // Rate of UTF-8 misses per UTF-8 flip
   double utf8_miss_rate(BenchmarkStage stage) const {
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    return 1;
+#else
     if (!has_events()) { return 1; }
     return double(utf8_miss[stage].best.branch_misses() - utf8[stage].best.branch_misses()) / double(utf8_miss.stats->blocks_with_utf8_flipped);
+#endif
   }
-
   // Extra cost of having escapes in a block
   double escape_cost(BenchmarkStage stage) const {
     return cost_per_block(stage, escape, escape.stats->blocks_with_escapes, struct7_full);
@@ -260,10 +275,15 @@ struct feature_benchmarker {
   }
   // Rate of escape misses per escape flip
   double escape_miss_rate(BenchmarkStage stage) const {
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    return 1;
+#else
     if (!has_events()) { return 1; }
     return double(escape_miss[stage].best.branch_misses() - escape[stage].best.branch_misses()) / double(escape_miss.stats->blocks_with_escapes_flipped);
+#endif
   }
 
+
   double calc_expected_feature_cost(BenchmarkStage stage, const benchmarker& file) const {
     // Expected base ns/block (empty)
     json_stats& stats = *file.stats;
@@ -300,7 +320,6 @@ struct feature_benchmarker {
   double calc_expected(BenchmarkStage stage, const benchmarker& file) const {
     return calc_expected_feature_cost(stage, file) + calc_expected_miss_cost(stage, file);
   }
-
   void print(const option_struct& options) const {
     printf("\n");
     printf("Features in ns/block (64 bytes):\n");
@@ -359,6 +378,22 @@ struct feature_benchmarker {
   }
 };
 
+#if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+void print_file_effectiveness(BenchmarkStage stage, const char* filename, const benchmarker& results, const feature_benchmarker& features) {
+  double actual = results[stage].best.elapsed_ns() / double(results.stats->blocks);
+  double calc = features.calc_expected(stage, results);
+  double calc_misses = features.calc_expected_misses(stage, results);
+  double calc_miss_cost = features.calc_expected_miss_cost(stage, results);
+  printf("        | %-8s ", benchmark_stage_name(stage));
+  printf("| %-15s ",   filename);
+  printf("|    %8.3g ", features.calc_expected_feature_cost(stage, results));
+  printf("|    %8.3g ", calc_miss_cost);
+  printf("| %8.3g ",  calc);
+  printf("| %8.3g ",  actual);
+  printf("| %+8.3g ", actual - calc);
+  printf("| %13llu ", (long long unsigned)(calc_misses));
+}
+#else
 void print_file_effectiveness(BenchmarkStage stage, const char* filename, const benchmarker& results, const feature_benchmarker& features) {
   double actual = results[stage].best.elapsed_ns() / double(results.stats->blocks);
   double calc = features.calc_expected(stage, results);
@@ -382,6 +417,7 @@ void print_file_effectiveness(BenchmarkStage stage, const char* filename, const
   }
   printf("|\n");
 }
+#endif
 
 int main(int argc, char *argv[]) {
   // Read options

benchmark/benchmarker.h

@@ -2,7 +2,7 @@
 #define __BENCHMARKER_H
 
 #include "event_counter.h"
-#include "simdjson.h" // For SIMDJSON_DISABLE_DEPRECATED_WARNINGS
+#include "simdjson.h"
 
 #include <cassert>
 #include <cctype>
@@ -423,7 +423,7 @@ struct benchmarker {
         stage.instructions() / static_cast<double>(stats->structurals),
         stage.instructions() / static_cast<double>(stage.cycles())
       );
-
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
       // NOTE: removed cycles/miss because it is a somewhat misleading stat
       printf("%s%-13s: %7.0f branch misses (%6.2f%%) - %.0f cache misses (%6.2f%%) - %.2f cache references\n",
         prefix,
@@ -434,6 +434,7 @@ struct benchmarker {
         percent(stage.cache_misses(), all_stages_without_allocation.cache_misses()),
         stage.cache_references()
       );
+#endif
     }
   }
 

benchmark/event_counter.h

@@ -1,6 +1,15 @@
 #ifndef __EVENT_COUNTER_H
 #define __EVENT_COUNTER_H
 
+#ifndef SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+#ifdef __aarch64__
+// on ARM, we use just cycles and instructions
+#define SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS 1
+#else
+// elsewhere, we try to use four counters.
+#define SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS 0
+#endif
+#endif
 #include <cassert>
 #include <cctype>
 #ifndef _MSC_VER
@@ -46,6 +55,12 @@ struct event_count {
   event_count(const event_count& other): elapsed(other.elapsed), event_counts(other.event_counts) { }
 
   // The types of counters (so we can read the getter more easily)
+  #if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+  enum event_counter_types {
+    CPU_CYCLES,
+    INSTRUCTIONS
+  };
+  #else
   enum event_counter_types {
     CPU_CYCLES,
     INSTRUCTIONS,
@@ -53,15 +68,16 @@ struct event_count {
     CACHE_REFERENCES,
     CACHE_MISSES
   };
-
+  #endif
   double elapsed_sec() const { return duration<double>(elapsed).count(); }
   double elapsed_ns() const { return duration<double, std::nano>(elapsed).count(); }
   double cycles() const { return static_cast<double>(event_counts[CPU_CYCLES]); }
   double instructions() const { return static_cast<double>(event_counts[INSTRUCTIONS]); }
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
   double branch_misses() const { return static_cast<double>(event_counts[BRANCH_MISSES]); }
   double cache_references() const { return static_cast<double>(event_counts[CACHE_REFERENCES]); }
   double cache_misses() const { return static_cast<double>(event_counts[CACHE_MISSES]); }
-
+#endif
   event_count& operator=(const event_count& other) {
     this->elapsed = other.elapsed;
     this->event_counts = other.event_counts;
@@ -105,9 +121,11 @@ struct event_aggregate {
   double elapsed_ns() const { return total.elapsed_ns() / iterations; }
   double cycles() const { return total.cycles() / iterations; }
   double instructions() const { return total.instructions() / iterations; }
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
   double branch_misses() const { return total.branch_misses() / iterations; }
   double cache_references() const { return total.cache_references() / iterations; }
   double cache_misses() const { return total.cache_misses() / iterations; }
+#endif
 };
 
 struct event_collector {
@@ -117,11 +135,16 @@ struct event_collector {
 #if defined(__linux__)
   LinuxEvents<PERF_TYPE_HARDWARE> linux_events;
   event_collector(bool quiet = false) : linux_events(vector<int>{
+  #if SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
+    PERF_COUNT_HW_CPU_CYCLES,
+    PERF_COUNT_HW_INSTRUCTIONS,
+  #else
     PERF_COUNT_HW_CPU_CYCLES,
     PERF_COUNT_HW_INSTRUCTIONS,
     PERF_COUNT_HW_BRANCH_MISSES,
     PERF_COUNT_HW_CACHE_REFERENCES,
     PERF_COUNT_HW_CACHE_MISSES
+  #endif
   }, quiet) {}
   bool has_events() {
     return linux_events.is_working();

benchmark/json_benchmark/run_json_benchmark.h

@@ -58,10 +58,11 @@ template<typename B, typename R> static void run_json_benchmark(benchmark::State
   if (collector.has_events()) {
     state.counters["instructions"] = events.instructions();
     state.counters["cycles"]       = events.cycles();
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
     state.counters["branch_miss"]  = events.branch_misses();
     state.counters["cache_miss"]   = events.cache_misses();
     state.counters["cache_ref"]    = events.cache_references();
-
+#endif
     state.counters["instructions_per_byte"]  = events.instructions() / double(bench.bytes_per_iteration());
     state.counters["instructions_per_cycle"] = events.instructions() / events.cycles();
     state.counters["cycles_per_byte"]        = events.cycles()       / double(bench.bytes_per_iteration());
@@ -69,9 +70,11 @@ template<typename B, typename R> static void run_json_benchmark(benchmark::State
 
     state.counters["best_instructions"] = events.best.instructions();
     state.counters["best_cycles"]       = events.best.cycles();
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
     state.counters["best_branch_miss"]  = events.best.branch_misses();
     state.counters["best_cache_miss"]   = events.best.cache_misses();
     state.counters["best_cache_ref"]    = events.best.cache_references();
+#endif
 
     state.counters["best_instructions_per_byte"]  = events.best.instructions() / double(bench.bytes_per_iteration());
     state.counters["best_instructions_per_cycle"] = events.best.instructions() / events.best.cycles();
@@ -92,9 +95,11 @@ template<typename B, typename R> static void run_json_benchmark(benchmark::State
   if (collector.has_events()) {
     label << " instructions=" << setw(12) << uint64_t(events.best.instructions())     << setw(0);
     label << " cycles="       << setw(12) << uint64_t(events.best.cycles())           << setw(0);
+#if !SIMDJSON_SIMPLE_PERFORMANCE_COUNTERS
     label << " branch_miss="  << setw(8)  << uint64_t(events.best.branch_misses())    << setw(0);
     label << " cache_miss="   << setw(8)  << uint64_t(events.best.cache_misses())     << setw(0);
     label << " cache_ref="    << setw(10) << uint64_t(events.best.cache_references()) << setw(0);
+#endif
   }
 
   label << " items=" << setw(10) << bench.items_per_iteration() << setw(0);

benchmark/large_amazon_cellphones/large_amazon_cellphones.h

@@ -6,6 +6,9 @@
 
 namespace large_amazon_cellphones {
 
+const bool UNTHREADED = false;
+const bool THREADED = true;
+
 static const simdjson::padded_string &get_built_json();
 
 using namespace json_benchmark;
@@ -81,11 +84,11 @@ static const simdjson::padded_string &get_built_json() {
   return json;
 }
 
-
+template<bool threaded>
 struct simdjson_dom;
 
 template<typename I> simdjson_really_inline static void large_amazon_cellphones(benchmark::State &state) {
-  run_json_benchmark<runner<I>, runner<simdjson_dom>>(state);
+  run_json_benchmark<runner<I>, runner<simdjson_dom<UNTHREADED>>>(state);
 }
 
 }   // namespace large_amazon_cellphones

benchmark/large_amazon_cellphones/simdjson_dom.h

@@ -9,12 +9,16 @@ namespace large_amazon_cellphones {
 
 using namespace simdjson;
 
+template<bool threaded>
 struct simdjson_dom {
   using StringType = std::string;
 
   dom::parser parser{};
 
   bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
+#ifdef SIMDJSON_THREADS_ENABLED
+    parser.threaded = threaded;
+#endif
     auto stream = parser.parse_many(json);
     auto i = stream.begin();
     ++i;  // Skip first line
@@ -38,7 +42,10 @@ struct simdjson_dom {
 
 };
 
-BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_dom)->UseManualTime();
+BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_dom<UNTHREADED>)->UseManualTime();
+#ifdef SIMDJSON_THREADS_ENABLED
+BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_dom<THREADED>)->UseManualTime();
+#endif
 
 } // namespace large_amazon_cellphones
 

benchmark/large_amazon_cellphones/simdjson_ondemand.h

@@ -8,12 +8,16 @@ namespace large_amazon_cellphones {
 
 using namespace simdjson;
 
+template<bool threaded>
 struct simdjson_ondemand {
   using StringType = std::string;
 
   ondemand::parser parser{};
 
   bool run(simdjson::padded_string &json, std::map<StringType, brand> &result) {
+#ifdef SIMDJSON_THREADS_ENABLED
+    parser.threaded = threaded;
+#endif
     ondemand::document_stream stream = parser.iterate_many(json);
     ondemand::document_stream::iterator i = stream.begin();
     ++i;  // Skip first line
@@ -58,7 +62,10 @@ struct simdjson_ondemand {
 
 };
 
-BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_ondemand)->UseManualTime();
+BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_ondemand<UNTHREADED>)->UseManualTime();
+#ifdef SIMDJSON_THREADS_ENABLED
+BENCHMARK_TEMPLATE(large_amazon_cellphones, simdjson_ondemand<THREADED>)->UseManualTime();
+#endif
 
 } // namespace amazon_cellphones
 

doc/basics.md

@@ -32,9 +32,12 @@ An overview of what you need to know to use simdjson, with examples.
 Requirements
 ------------------
 
-- A recent compiler (LLVM clang6 or better, GNU GCC 7.4 or better, Xcode 11 or better) on a 64-bit (PPC, ARM or x64 Intel/AMD) POSIX systems such as macOS, freeBSD or Linux. We require that the compiler supports the C++11 standard or better.
+- A recent compiler (LLVM clang 6 or better, GNU GCC 7.4 or better, Xcode 11 or better) on a 64-bit (PPC, ARM or x64 Intel/AMD) POSIX systems such as macOS, freeBSD or Linux. We require that the compiler supports the C++11 standard or better.
 - Visual Studio 2017 or better under 64-bit Windows. Users should target a 64-bit build (x64) instead of a 32-bit build (x86). We support the LLVM clang compiler under Visual Studio (clangcl) as well as as the regular Visual Studio compiler. We also support MinGW 64-bit under Windows.
 
+
+Support for AVX-512 require a processor with AVX512-VBMI2 support (Ice Lake or better) under a 64-bit system and a recent compiler (LLVM clang 6 or better, GCC 8 or better, Visual Studio 2019 or better).
+
 Including simdjson
 ------------------
 
@@ -1401,10 +1404,6 @@ You must check the type before accessing the value: it is an error to call `get_
 
 The `get_number()` function is designed with performance in mind. When calling `get_number()`, you scan the number string only once, determining efficiently the type and storing it in an efficient manner.
 
-If you only need to compute `v.get_number().get_number_type()` on
-a `document` or `value` instance, you should call directly the faster method
-`v.get_number_type()` which does not generate an
-intermediate `number` instance.
 
 Consider the following example:
 ```C++
@@ -1417,9 +1416,7 @@ Consider the following example:
       std::cout << "negative: " << val.is_negative() << " ";
       std::cout << "is_integer: " << val.is_integer() << " ";
       ondemand::number num = val.get_number();
-      // direct computation without materializing the number:
-      ondemand::number_type dt = val.get_number_type();
-      if(t != dt) { throw std::runtime_error("bug"); }
+      ondemand::number_type t = num.get_number_type();
       switch(t) {
         case ondemand::number_type::signed_integer:
           std::cout  << "integer: " << int64_t(num) << " ";

include/simdjson/arm64/implementation.h

@@ -22,6 +22,7 @@ public:
   ) const noexcept final;
   simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
   simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final;
 };
 
 } // namespace arm64

include/simdjson/common_defs.h

@@ -122,6 +122,9 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
 
   #define SIMDJSON_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push")
   // gcc doesn't seem to disable all warnings with all and extra, add warnings here as necessary
+  // We do it separately for clang since it has different warnings.
+  #ifdef __clang__
+  // clang is missing -Wmaybe-uninitialized.
   #define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \
     SIMDJSON_DISABLE_GCC_WARNING(-Weffc++) \
     SIMDJSON_DISABLE_GCC_WARNING(-Wall) \
@@ -134,6 +137,22 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
     SIMDJSON_DISABLE_GCC_WARNING(-Wshadow) \
     SIMDJSON_DISABLE_GCC_WARNING(-Wunused-parameter) \
     SIMDJSON_DISABLE_GCC_WARNING(-Wunused-variable)
+  #else // __clang__
+  #define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \
+    SIMDJSON_DISABLE_GCC_WARNING(-Weffc++) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wall) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wconversion) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wextra) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wattributes) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wreturn-type) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wshadow) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wunused-parameter) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wunused-variable) \
+    SIMDJSON_DISABLE_GCC_WARNING(-Wmaybe-uninitialized)
+  #endif // __clang__
+
   #define SIMDJSON_PRAGMA(P) _Pragma(#P)
   #define SIMDJSON_DISABLE_GCC_WARNING(WARNING) SIMDJSON_PRAGMA(GCC diagnostic ignored #WARNING)
   #if defined(SIMDJSON_CLANG_VISUAL_STUDIO)

include/simdjson/fallback/implementation.h

@@ -25,6 +25,7 @@ public:
   ) const noexcept final;
   simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
   simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final;
 };
 
 } // namespace fallback

include/simdjson/generic/numberparsing.h

@@ -1067,6 +1067,11 @@ simdjson_unused simdjson_really_inline simdjson_result<ondemand::number_type> ge
   while(static_cast<uint8_t>(*p - '0') <= 9) { p++; }
   if ( p == src ) { return NUMBER_ERROR; }
   if (jsoncharutils::is_structural_or_whitespace(*p)) {
+    // We have an integer.
+    // If the number is negative and valid, it must be a signed integer.
+    if(negative) { return ondemand::number_type::signed_integer; }
+    // We want values larger or equal to 9223372036854775808 to be unsigned
+    // integers, and the other values to be signed integers.
     int digit_count = int(p - src);
     if(digit_count >= 19) {
       const uint8_t * smaller_big_integer = reinterpret_cast<const uint8_t *>("9223372036854775808");
@@ -1076,6 +1081,7 @@ simdjson_unused simdjson_really_inline simdjson_result<ondemand::number_type> ge
     }
     return ondemand::number_type::signed_integer;
   }
+  // Hopefully, we have 'e' or 'E' or '.'.
   return ondemand::number_type::floating_point_number;
 }
 

include/simdjson/generic/ondemand/array-inl.h

@@ -89,6 +89,7 @@ simdjson_really_inline simdjson_result<std::string_view> array::raw_json() noexc
   return std::string_view(reinterpret_cast<const char*>(starting_point), size_t(final_point - starting_point));
 }
 
+SIMDJSON_PUSH_DISABLE_WARNINGS
 SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING
 simdjson_really_inline simdjson_result<size_t> array::count_elements() & noexcept {
   size_t count{0};
@@ -101,6 +102,7 @@ simdjson_really_inline simdjson_result<size_t> array::count_elements() & noexcep
   iter.reset_array();
   return count;
 }
+SIMDJSON_POP_DISABLE_WARNINGS
 
 simdjson_really_inline simdjson_result<bool> array::is_empty() & noexcept {
   bool is_not_empty;

include/simdjson/generic/ondemand/document-inl.h

@@ -24,6 +24,10 @@ inline simdjson_result<const char *> document::current_location() noexcept {
   return iter.current_location();
 }
 
+inline int32_t document::current_depth() const noexcept {
+  return iter.depth();
+}
+
 inline bool document::is_alive() noexcept {
   return iter.is_alive();
 }
@@ -471,6 +475,11 @@ simdjson_really_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IM
   return first.current_location();
 }
 
+simdjson_really_inline int32_t simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::current_depth() const noexcept {
+  if (error()) { return error(); }
+  return first.current_depth();
+}
+
 simdjson_really_inline simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::raw_json_token() noexcept {
   if (error()) { return error(); }
   return first.raw_json_token();
@@ -528,6 +537,7 @@ simdjson_really_inline simdjson_result<value> document_reference::find_field_uno
 simdjson_really_inline simdjson_result<json_type> document_reference::type() noexcept { return doc->type(); }
 simdjson_really_inline simdjson_result<bool> document_reference::is_scalar() noexcept { return doc->is_scalar(); }
 simdjson_really_inline simdjson_result<const char *> document_reference::current_location() noexcept { return doc->current_location(); }
+simdjson_really_inline int32_t document_reference::current_depth() const noexcept { return doc->current_depth(); }
 simdjson_really_inline bool document_reference::is_negative() noexcept { return doc->is_negative(); }
 simdjson_really_inline simdjson_result<bool> document_reference::is_integer() noexcept { return doc->is_integer(); }
 simdjson_really_inline simdjson_result<number_type> document_reference::get_number_type() noexcept { return doc->get_number_type(); }

include/simdjson/generic/ondemand/document.h

@@ -399,7 +399,14 @@ public:
    */
   simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
   /**
-   * Determine the number type (integer or floating-point number).
+   * Determine the number type (integer or floating-point number) as quickly
+   * as possible. This function does not fully validate the input. It is
+   * useful when you only need to classify the numbers, without parsing them.
+   *
+   * If you are planning to retrieve the value or you need full validation,
+   * consider using the get_number() method instead: it will fully parse
+   * and validate the input, and give you access to the type:
+   * get_number().get_number_type().
    *
    * get_number_type() is number_type::unsigned_integer if we have
    * an integer greater or equal to 9223372036854775808
@@ -407,8 +414,7 @@ public:
    * integer that is less than 9223372036854775808
    * Otherwise, get_number_type() has value number_type::floating_point_number
    *
-   * This function req
-   * uires processing the number string, but it is expected
+   * This function requires processing the number string, but it is expected
    * to be faster than get_number().get_number_type() because it is does not
    * parse the number value.
    *
@@ -489,6 +495,17 @@ public:
    */
   inline simdjson_result<const char *> current_location() noexcept;
 
+  /**
+   * Returns the current depth in the document if in bounds.
+   *
+   * E.g.,
+   *  0 = finished with document
+   *  1 = document root value (could be [ or {, not yet known)
+   *  2 = , or } inside root array/object
+   *  3 = key or value inside root array/object.
+   */
+  simdjson_really_inline int32_t current_depth() const noexcept;
+
   /**
    * Get the value associated with the given JSON pointer.  We use the RFC 6901
    * https://tools.ietf.org/html/rfc6901 standard.
@@ -609,6 +626,7 @@ public:
   simdjson_really_inline simdjson_result<bool> is_scalar() noexcept;
 
   simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
+  simdjson_really_inline int32_t current_depth() const noexcept;
   simdjson_really_inline bool is_negative() noexcept;
   simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
   simdjson_really_inline simdjson_result<number_type> get_number_type() noexcept;
@@ -675,6 +693,7 @@ public:
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_type> type() noexcept;
   simdjson_really_inline simdjson_result<bool> is_scalar() noexcept;
   simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
+  simdjson_really_inline int32_t current_depth() const noexcept;
   simdjson_really_inline bool is_negative() noexcept;
   simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> get_number_type() noexcept;
@@ -735,6 +754,7 @@ public:
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_type> type() noexcept;
   simdjson_really_inline simdjson_result<bool> is_scalar() noexcept;
   simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
+  simdjson_really_inline int32_t current_depth() const noexcept;
   simdjson_really_inline bool is_negative() noexcept;
   simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number_type> get_number_type() noexcept;

include/simdjson/generic/ondemand/json_iterator-inl.h

@@ -47,6 +47,27 @@ inline void json_iterator::rewind() noexcept {
   _depth = 1;
 }
 
+inline bool json_iterator::balanced() const noexcept {
+  token_iterator ti(token);
+  int32_t count{0};
+  ti.set_position( root_position() );
+  while(ti.peek() <= peek_last()) {
+    switch (*ti.return_current_and_advance())
+    {
+    case '[': case '{':
+      count++;
+      break;
+    case ']': case '}':
+      count--;
+      break;
+    default:
+      break;
+    }
+  }
+  return count == 0;
+}
+
+
 // GCC 7 warns when the first line of this function is inlined away into oblivion due to the caller
 // relating depth and parent_depth, which is a desired effect. The warning does not show up if the
 // skip_child() function is not marked inline).

include/simdjson/generic/ondemand/json_iterator.h

@@ -154,7 +154,7 @@ public:
    * This is not null-terminated; it is a view into the JSON.
    *
    * You may be pointing outside of the input buffer: it is not generally
-   * safe to derefence this pointer.
+   * safe to dereference this pointer.
    */
   simdjson_really_inline const uint8_t *unsafe_pointer() const noexcept;
   /**
@@ -251,6 +251,13 @@ public:
    * as if it had just been created.
    */
   inline void rewind() noexcept;
+  /**
+   * This checks whether the {,},[,] are balanced so that the document
+   * ends with proper zero depth. This requires scanning the whole document
+   * and it may be expensive. It is expected that it will be rarely called.
+   * It does not attempt to match { with } and [ with ].
+   */
+  inline bool balanced() const noexcept;
 protected:
   simdjson_really_inline json_iterator(const uint8_t *buf, ondemand::parser *parser) noexcept;
   /// The last token before the end

include/simdjson/generic/ondemand/logger-inl.h

@@ -85,7 +85,7 @@ inline void log_headers() noexcept {
       printf("# skip says 'this is a structural or value I am skipping'\n");
       printf("# +/-skip says 'this is a start/end array or object I am skipping'\n");
       printf("#\n");
-      printf("# The identation of the terms (array, string,...) indicates the depth,\n");
+      printf("# The indentation of the terms (array, string,...) indicates the depth,\n");
       printf("# in addition to the depth being displayed.\n");
       printf("#\n");
       printf("# Every token in the document has a single depth determined by the tokens before it,\n");

include/simdjson/generic/ondemand/object.h

@@ -118,7 +118,7 @@ public:
    * instance: there is no rewind and no invalidation.
    *
    * You may call at_pointer more than once on an object, but each time the pointer is advanced
-   * to be within the value matched by the key indicated by the JSON pointer query. Thus any preceeding
+   * to be within the value matched by the key indicated by the JSON pointer query. Thus any preceding
    * key (as well as the current key) can no longer be used with following JSON pointer calls.
    *
    * Also note that at_pointer() relies on find_field() which implies that we do not unescape keys when matching.
@@ -168,7 +168,7 @@ public:
   simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
   /**
    * Consumes the object and returns a string_view instance corresponding to the
-   * object as represented in JSON. It points inside the original byte array containg
+   * object as represented in JSON. It points inside the original byte array containing
    * the JSON document.
    */
   simdjson_really_inline simdjson_result<std::string_view> raw_json() noexcept;

include/simdjson/generic/ondemand/object_iterator-inl.h

@@ -26,6 +26,8 @@ simdjson_really_inline bool object_iterator::operator!=(const object_iterator &)
   return iter.is_open();
 }
 
+SIMDJSON_PUSH_DISABLE_WARNINGS
+SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING
 simdjson_really_inline object_iterator &object_iterator::operator++() noexcept {
   // TODO this is a safety rail ... users should exit loops as soon as they receive an error.
   // Nonetheless, let's see if performance is OK with this if statement--the compiler may give it to us for free.
@@ -38,6 +40,7 @@ simdjson_really_inline object_iterator &object_iterator::operator++() noexcept {
   if ((error = iter.has_next_field().get(has_value) )) { return *this; };
   return *this;
 }
+SIMDJSON_POP_DISABLE_WARNINGS
 
 //
 // ### Live States

include/simdjson/generic/ondemand/raw_json_string-inl.h

@@ -7,7 +7,7 @@ simdjson_really_inline raw_json_string::raw_json_string(const uint8_t * _buf) no
 
 simdjson_really_inline const char * raw_json_string::raw() const noexcept { return reinterpret_cast<const char *>(buf); }
 simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape(uint8_t *&dst) const noexcept {
-  uint8_t *end = stringparsing::parse_string(buf, dst);
+  uint8_t *end = parse_string(buf, dst);
   if (!end) { return STRING_ERROR; }
   std::string_view result(reinterpret_cast<const char *>(dst), end-dst);
   dst = end;

include/simdjson/generic/ondemand/raw_json_string.h

@@ -116,6 +116,26 @@ public:
   static simdjson_really_inline bool is_free_from_unescaped_quote(std::string_view target) noexcept;
   static simdjson_really_inline bool is_free_from_unescaped_quote(const char* target) noexcept;
 
+  /**
+   * Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer.
+   * The provided pointer is advanced to the end of the string by reference, and a string_view instance
+   * is returned. You can ensure that your buffer is large enough by allocating a block of memory at least
+   * as large as the input JSON plus SIMDJSON_PADDING and then unescape all strings to this one buffer.
+   *
+   * This unescape function is a low-level function. If you want a more user-friendly approach, you should
+   * avoid raw_json_string instances (e.g., by calling unescaped_key() instead of key() or get_string()
+   * instead of get_raw_json_string()).
+   *
+   * ## IMPORTANT: string_view lifetime
+   *
+   * The string_view is only valid as long as the bytes in dst.
+   *
+   * @param dst A pointer to a buffer at least large enough to write this string as well as
+   *            an additional SIMDJSON_PADDING bytes.
+   * @return A string_view pointing at the unescaped string in dst
+   * @error STRING_ERROR if escapes are incorrect.
+   */
+  simdjson_really_inline simdjson_result<std::string_view> unescape(uint8_t *&dst) const noexcept;
 private:
 
 
@@ -130,20 +150,6 @@ private:
    */
   simdjson_really_inline simdjson_warn_unused bool alive() const noexcept { return buf != nullptr; }
 
-  /**
-   * Unescape this JSON string, replacing \\ with \, \n with newline, etc.
-   *
-   * ## IMPORTANT: string_view lifetime
-   *
-   * The string_view is only valid as long as the bytes in dst.
-   *
-   * @param dst A pointer to a buffer at least large enough to write this string as well as a \0.
-   *            dst will be updated to the next unused location (just after the \0 written out at
-   *            the end of this string).
-   * @return A string_view pointing at the unescaped string in dst
-   * @error STRING_ERROR if escapes are incorrect.
-   */
-  simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(uint8_t *&dst) const noexcept;
   /**
    * Unescape this JSON string, replacing \\ with \, \n with newline, etc.
    *

include/simdjson/generic/ondemand/value-inl.h

@@ -181,6 +181,10 @@ simdjson_really_inline simdjson_result<const char *> value::current_location() n
   return iter.json_iter().current_location();
 }
 
+simdjson_really_inline int32_t value::current_depth() const noexcept{
+  return iter.json_iter().depth();
+}
+
 simdjson_really_inline simdjson_result<value> value::at_pointer(std::string_view json_pointer) noexcept {
   json_type t;
   SIMDJSON_TRY(type().get(t));
@@ -400,6 +404,11 @@ simdjson_really_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IM
   return first.current_location();
 }
 
+simdjson_really_inline int32_t simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::current_depth() const noexcept {
+  if (error()) { return error(); }
+  return first.current_depth();
+}
+
 simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::at_pointer(std::string_view json_pointer) noexcept {
   if (error()) { return error(); }
   return first.at_pointer(json_pointer);

include/simdjson/generic/ondemand/value.h

@@ -369,7 +369,14 @@ public:
    */
   simdjson_really_inline simdjson_result<bool> is_integer() noexcept;
   /**
-   * Determine the number type (integer or floating-point number).
+   * Determine the number type (integer or floating-point number) as quickly
+   * as possible. This function does not fully validate the input. It is
+   * useful when you only need to classify the numbers, without parsing them.
+   *
+   * If you are planning to retrieve the value or you need full validation,
+   * consider using the get_number() method instead: it will fully parse
+   * and validate the input, and give you access to the type:
+   * get_number().get_number_type().
    *
    * get_number_type() is number_type::unsigned_integer if we have
    * an integer greater or equal to 9223372036854775808
@@ -448,6 +455,17 @@ public:
    */
   simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
 
+  /**
+   * Returns the current depth in the document if in bounds.
+   *
+   * E.g.,
+   *  0 = finished with document
+   *  1 = document root value (could be [ or {, not yet known)
+   *  2 = , or } inside root array/object
+   *  3 = key or value inside root array/object.
+   */
+  simdjson_really_inline int32_t current_depth() const noexcept;
+
   /**
    * Get the value associated with the given JSON pointer.  We use the RFC 6901
    * https://tools.ietf.org/html/rfc6901 standard.
@@ -479,7 +497,7 @@ public:
    * to call at_pointer on the same array.
    *
    * You may call at_pointer more than once on an object, but each time the pointer is advanced
-   * to be within the value matched by the key indicated by the JSON pointer query. Thus any preceeding
+   * to be within the value matched by the key indicated by the JSON pointer query. Thus any preceding
    * key (as well as the current key) can no longer be used with following JSON pointer calls.
    *
    * Also note that at_pointer() relies on find_field() which implies that we do not unescape keys when matching
@@ -650,7 +668,8 @@ public:
 
   /** @copydoc simdjson_really_inline simdjson_result<const char *> current_location() noexcept */
   simdjson_really_inline simdjson_result<const char *> current_location() noexcept;
-
+  /** @copydoc simdjson_really_inline int32_t current_depth() const noexcept */
+  simdjson_really_inline int32_t current_depth() const noexcept;
   simdjson_really_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at_pointer(std::string_view json_pointer) noexcept;
 };
 

include/simdjson/generic/ondemand/value_iterator-inl.h

@@ -39,9 +39,22 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
   // current document. It only works in the normal mode where we have indexed a single document.
   // Note that adding a check for 'streaming' is not expensive since we only have at most
   // one root element.
-  if (! _json_iter->streaming() && (*_json_iter->peek_last() != '}')) {
-    _json_iter->abandon();
-    return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing } at end");
+  if ( ! _json_iter->streaming() ) {
+    if (*_json_iter->peek_last() != '}') {
+      _json_iter->abandon();
+      return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing } at end");
+    }
+    // If the last character is } *and* the first gibberish character is also '}'
+    // then on-demand could accidentally go over. So we need additional checks.
+    // https://github.com/simdjson/simdjson/issues/1834
+    // Checking that the document is balanced requires a full scan which is potentially
+    // expensive, but it only happens in edge cases where the first padding character is
+    // a closing bracket.
+    if ((*_json_iter->peek(_json_iter->end_position()) == '}') && (!_json_iter->balanced())) {
+      _json_iter->abandon();
+      // The exact error would require more work. It will typically be an unclosed object.
+      return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "the document is unbalanced");
+    }
   }
   return started_object();
 }
@@ -167,6 +180,8 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
   return false;
 }
 
+SIMDJSON_PUSH_DISABLE_WARNINGS
+SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING
 simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::find_field_unordered_raw(const std::string_view key) noexcept {
   /**
    * When find_field_unordered_raw is called, we can either be pointing at the
@@ -354,6 +369,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
   // never reach this point.
   return false;
 }
+SIMDJSON_POP_DISABLE_WARNINGS
 
 simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> value_iterator::field_key() noexcept {
   assert_at_next();
@@ -408,9 +424,22 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
   // current document. It only works in the normal mode where we have indexed a single document.
   // Note that adding a check for 'streaming' is not expensive since we only have at most
   // one root element.
-  if ( ! _json_iter->streaming() && (*_json_iter->peek_last() != ']')) {
-    _json_iter->abandon();
-    return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing ] at end");
+  if ( ! _json_iter->streaming() ) {
+    if (*_json_iter->peek_last() != ']') {
+      _json_iter->abandon();
+      return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing ] at end");
+    }
+    // If the last character is ] *and* the first gibberish character is also ']'
+    // then on-demand could accidentally go over. So we need additional checks.
+    // https://github.com/simdjson/simdjson/issues/1834
+    // Checking that the document is balanced requires a full scan which is potentially
+    // expensive, but it only happens in edge cases where the first padding character is
+    // a closing bracket.
+    if ((*_json_iter->peek(_json_iter->end_position()) == ']') && (!_json_iter->balanced())) {
+      _json_iter->abandon();
+      // The exact error would require more work. It will typically be an unclosed array.
+      return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "the document is unbalanced");
+    }
   }
   return started_array();
 }

include/simdjson/generic/ondemand/value_iterator.h

@@ -76,7 +76,7 @@ public:
   /**
    * Get the depth of this value.
    */
-  simdjson_really_inline depth_t depth() const noexcept;
+  simdjson_really_inline int32_t depth() const noexcept;
 
   /**
    * Get the JSON type of this value.

include/simdjson/generic/stringparsing.h

@@ -1,138 +0,0 @@
-// This file contains the common code every implementation uses
-// It is intended to be included multiple times and compiled multiple times
-
-namespace simdjson {
-namespace SIMDJSON_IMPLEMENTATION {
-namespace {
-/// @private
-namespace stringparsing {
-
-// begin copypasta
-// These chars yield themselves: " \ /
-// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab
-// u not handled in this table as it's complex
-static const uint8_t escape_map[256] = {
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0, // 0x0.
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-    0, 0, 0x22, 0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0x2f,
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0, // 0x4.
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0x5c, 0, 0,    0, // 0x5.
-    0, 0, 0x08, 0, 0,    0, 0x0c, 0, 0, 0, 0, 0, 0,    0, 0x0a, 0, // 0x6.
-    0, 0, 0x0d, 0, 0x09, 0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0, // 0x7.
-
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
-};
-
-// handle a unicode codepoint
-// write appropriate values into dest
-// src will advance 6 bytes or 12 bytes
-// dest will advance a variable amount (return via pointer)
-// return true if the unicode codepoint was valid
-// We work in little-endian then swap at write time
-simdjson_warn_unused
-simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
-                                            uint8_t **dst_ptr) {
-  // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
-  // conversion isn't valid; we defer the check for this to inside the
-  // multilingual plane check
-  uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
-  *src_ptr += 6;
-  // check for low surrogate for characters outside the Basic
-  // Multilingual Plane.
-  if (code_point >= 0xd800 && code_point < 0xdc00) {
-    if (((*src_ptr)[0] != '\\') || (*src_ptr)[1] != 'u') {
-      return false;
-    }
-    uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
-
-    // if the first code point is invalid we will get here, as we will go past
-    // the check for being outside the Basic Multilingual plane. If we don't
-    // find a \u immediately afterwards we fail out anyhow, but if we do,
-    // this check catches both the case of the first code point being invalid
-    // or the second code point being invalid.
-    if ((code_point | code_point_2) >> 16) {
-      return false;
-    }
-
-    code_point =
-        (((code_point - 0xd800) << 10) | (code_point_2 - 0xdc00)) + 0x10000;
-    *src_ptr += 6;
-  }
-  size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr);
-  *dst_ptr += offset;
-  return offset > 0;
-}
-
-/**
- * Unescape a string from src to dst, stopping at a final unescaped quote. E.g., if src points at 'joe"', then
- * dst needs to have four free bytes.
- */
-simdjson_warn_unused simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) {
-  while (1) {
-    // Copy the next n bytes, and find the backslash and quote in them.
-    auto bs_quote = backslash_and_quote::copy_and_find(src, dst);
-    // If the next thing is the end quote, copy and return
-    if (bs_quote.has_quote_first()) {
-      // we encountered quotes first. Move dst to point to quotes and exit
-      return dst + bs_quote.quote_index();
-    }
-    if (bs_quote.has_backslash()) {
-      /* find out where the backspace is */
-      auto bs_dist = bs_quote.backslash_index();
-      uint8_t escape_char = src[bs_dist + 1];
-      /* we encountered backslash first. Handle backslash */
-      if (escape_char == 'u') {
-        /* move src/dst up to the start; they will be further adjusted
-           within the unicode codepoint handling code. */
-        src += bs_dist;
-        dst += bs_dist;
-        if (!handle_unicode_codepoint(&src, &dst)) {
-          return nullptr;
-        }
-      } else {
-        /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and
-         * write bs_dist+1 characters to output
-         * note this may reach beyond the part of the buffer we've actually
-         * seen. I think this is ok */
-        uint8_t escape_result = escape_map[escape_char];
-        if (escape_result == 0u) {
-          return nullptr; /* bogus escape value is an error */
-        }
-        dst[bs_dist] = escape_result;
-        src += bs_dist + 2;
-        dst += bs_dist + 1;
-      }
-    } else {
-      /* they are the same. Since they can't co-occur, it means we
-       * encountered neither. */
-      src += backslash_and_quote::BYTES_PROCESSED;
-      dst += backslash_and_quote::BYTES_PROCESSED;
-    }
-  }
-  /* can't be reached */
-  return nullptr;
-}
-
-simdjson_unused simdjson_warn_unused simdjson_really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *&current_string_buf_loc, std::string_view &s) {
-  if (*(src++) != '"') { return STRING_ERROR; }
-  auto end = stringparsing::parse_string(src, current_string_buf_loc);
-  if (!end) { return STRING_ERROR; }
-  s = std::string_view(reinterpret_cast<const char *>(current_string_buf_loc), end-current_string_buf_loc);
-  current_string_buf_loc = end;
-  return SUCCESS;
-}
-
-} // namespace stringparsing
-} // unnamed namespace
-} // namespace SIMDJSON_IMPLEMENTATION
-} // namespace simdjson

include/simdjson/haswell/implementation.h

@@ -23,6 +23,7 @@ public:
   ) const noexcept final;
   simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
   simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final;
 };
 
 } // namespace haswell

include/simdjson/icelake/implementation.h

@@ -23,6 +23,7 @@ public:
   ) const noexcept final;
   simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
   simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final;
 };
 
 } // namespace icelake

include/simdjson/icelake/simd.h

@@ -3,6 +3,32 @@
 
 #include "simdjson/internal/simdprune_tables.h"
 
+
+
+#if defined(__GNUC__) && !defined(__clang__)
+#if __GNUC__ == 8
+#define SIMDJSON_GCC8 1
+#endif //  __GNUC__ == 8
+#endif // defined(__GNUC__) && !defined(__clang__)
+
+#if SIMDJSON_GCC8
+/**
+ * GCC 8 fails to provide _mm512_set_epi8. We roll our own.
+ */
+inline __m512i _mm512_set_epi8(uint8_t a0, uint8_t a1, uint8_t a2, uint8_t a3, uint8_t a4, uint8_t a5, uint8_t a6, uint8_t a7, uint8_t a8, uint8_t a9, uint8_t a10, uint8_t a11, uint8_t a12, uint8_t a13, uint8_t a14, uint8_t a15, uint8_t a16, uint8_t a17, uint8_t a18, uint8_t a19, uint8_t a20, uint8_t a21, uint8_t a22, uint8_t a23, uint8_t a24, uint8_t a25, uint8_t a26, uint8_t a27, uint8_t a28, uint8_t a29, uint8_t a30, uint8_t a31, uint8_t a32, uint8_t a33, uint8_t a34, uint8_t a35, uint8_t a36, uint8_t a37, uint8_t a38, uint8_t a39, uint8_t a40, uint8_t a41, uint8_t a42, uint8_t a43, uint8_t a44, uint8_t a45, uint8_t a46, uint8_t a47, uint8_t a48, uint8_t a49, uint8_t a50, uint8_t a51, uint8_t a52, uint8_t a53, uint8_t a54, uint8_t a55, uint8_t a56, uint8_t a57, uint8_t a58, uint8_t a59, uint8_t a60, uint8_t a61, uint8_t a62, uint8_t a63) {
+  return _mm512_set_epi64(uint64_t(a7) + (uint64_t(a6) << 8) + (uint64_t(a5) << 16) + (uint64_t(a4) << 24) + (uint64_t(a3) << 32) + (uint64_t(a2) << 40) + (uint64_t(a1) << 48) + (uint64_t(a0) << 56),
+                          uint64_t(a15) + (uint64_t(a14) << 8) + (uint64_t(a13) << 16) + (uint64_t(a12) << 24) + (uint64_t(a11) << 32) + (uint64_t(a10) << 40) + (uint64_t(a9) << 48) + (uint64_t(a8) << 56),
+                          uint64_t(a23) + (uint64_t(a22) << 8) + (uint64_t(a21) << 16) + (uint64_t(a20) << 24) + (uint64_t(a19) << 32) + (uint64_t(a18) << 40) + (uint64_t(a17) << 48) + (uint64_t(a16) << 56),
+                          uint64_t(a31) + (uint64_t(a30) << 8) + (uint64_t(a29) << 16) + (uint64_t(a28) << 24) + (uint64_t(a27) << 32) + (uint64_t(a26) << 40) + (uint64_t(a25) << 48) + (uint64_t(a24) << 56),
+                          uint64_t(a39) + (uint64_t(a38) << 8) + (uint64_t(a37) << 16) + (uint64_t(a36) << 24) + (uint64_t(a35) << 32) + (uint64_t(a34) << 40) + (uint64_t(a33) << 48) + (uint64_t(a32) << 56),
+                          uint64_t(a47) + (uint64_t(a46) << 8) + (uint64_t(a45) << 16) + (uint64_t(a44) << 24) + (uint64_t(a43) << 32) + (uint64_t(a42) << 40) + (uint64_t(a41) << 48) + (uint64_t(a40) << 56),
+                          uint64_t(a55) + (uint64_t(a54) << 8) + (uint64_t(a53) << 16) + (uint64_t(a52) << 24) + (uint64_t(a51) << 32) + (uint64_t(a50) << 40) + (uint64_t(a49) << 48) + (uint64_t(a48) << 56),
+                          uint64_t(a63) + (uint64_t(a62) << 8) + (uint64_t(a61) << 16) + (uint64_t(a60) << 24) + (uint64_t(a59) << 32) + (uint64_t(a58) << 40) + (uint64_t(a57) << 48) + (uint64_t(a56) << 56));
+}
+#endif // SIMDJSON_GCC8
+
+
+
 namespace simdjson {
 namespace SIMDJSON_IMPLEMENTATION {
 namespace {
@@ -45,13 +71,21 @@ namespace simd {
     simdjson_really_inline base8() : base<simd8<T>>() {}
     simdjson_really_inline base8(const __m512i _value) : base<simd8<T>>(_value) {}
 
-    simdjson_really_inline uint64_t operator==(const simd8<T> other) const { return _mm512_cmpeq_epi8_mask(*this, other); }
+    friend simdjson_really_inline uint64_t operator==(const simd8<T> lhs, const simd8<T> rhs) {
+      return _mm512_cmpeq_epi8_mask(lhs, rhs);
+    }
 
     static const int SIZE = sizeof(base<T>::value);
 
     template<int N=1>
     simdjson_really_inline simd8<T> prev(const simd8<T> prev_chunk) const {
+#if SIMDJSON_GCC8
+     // workaround for compilers unable to figure out that 16 - N is a constant (GCC 8)
+      constexpr int shift = 16 - N;
+      return _mm512_alignr_epi8(*this, _mm512_permutex2var_epi64(prev_chunk, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), *this), shift);
+#else
       return _mm512_alignr_epi8(*this, _mm512_permutex2var_epi64(prev_chunk, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), *this), 16 - N);
+#endif
     }
   };
 
@@ -328,8 +362,6 @@ namespace simd {
       const simd8<T> mask = simd8<T>::splat(m);
       return this->chunks[0] <= mask;
     }
-    simdjson_really_inline operator __m512i() const { return __m512i(this->chunks[0]); }
-
   }; // struct simd8x64<T>
 
 } // namespace simd

include/simdjson/implementation.h

@@ -18,8 +18,6 @@ namespace simdjson {
  * @return true if the string is valid UTF-8.
  */
 simdjson_warn_unused bool validate_utf8(const char * buf, size_t len) noexcept;
-
-
 /**
  * Validate the UTF-8 string.
  *
@@ -40,6 +38,20 @@ simdjson_really_inline simdjson_warn_unused bool validate_utf8(const std::string
   return validate_utf8(s.data(), s.size());
 }
 
+/**
+ * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There
+ * must be an unescaped quote terminating the string. It returns the final output
+ * position as pointer. In case of error (e.g., the string has bad escaped codes),
+ * then null_nullptrptr is returned. It is assumed that the output buffer is large
+ * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes +
+ * SIMDJSON_PADDING bytes.
+ *
+ * @param str pointer to the beginning of a valid UTF-8 JSON string, must end with an unescaped quote.
+ * @param dst pointer to a destination buffer, it must point a region in memory of sufficient size.
+ * @return end of the of the written region (exclusive) or nullptr in case of error.
+ */
+simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) noexcept;
+
 namespace dom {
   class document;
 } // namespace dom
@@ -136,6 +148,22 @@ public:
    */
   simdjson_warn_unused virtual bool validate_utf8(const char *buf, size_t len) const noexcept = 0;
 
+  /**
+   * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There
+   * must be an unescaped quote terminating the string. It returns the final output
+   * position as pointer. In case of error (e.g., the string has bad escaped codes),
+   * then null_nullptrptr is returned. It is assumed that the output buffer is large
+   * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes +
+   * SIMDJSON_PADDING bytes.
+   *
+   * Overridden by each implementation.
+   *
+   * @param str pointer to the beginning of a valid UTF-8 JSON string, must end with an unescaped quote.
+   * @param dst pointer to a destination buffer, it must point a region in memory of sufficient size.
+   * @return end of the of the written region (exclusive) or nullptr in case of error.
+   */
+  simdjson_warn_unused virtual uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept = 0;
+
 protected:
   /** @private Construct an implementation with the given name and description. For subclasses. */
   simdjson_really_inline implementation(

include/simdjson/implementations.h

@@ -21,6 +21,19 @@
 #endif
 #endif
 
+#ifdef _MSC_VER
+#if _MSC_VER >= 1920
+// Visual Studio 2019 and up support VBMI2 under x64 even if the header
+// avx512vbmi2intrin.h is not found.
+#define SIMDJSON_COMPILER_SUPPORTS_VBMI2 1
+#endif
+#endif
+
+// By default, we allow AVX512.
+#ifndef SIMDJSON_AVX512_ALLOWED
+#define SIMDJSON_AVX512_ALLOWED 1
+#endif
+
 // Default Icelake to on if this is x86-64. Even if we're not compiled for it, it could be selected
 // at runtime.
 #ifndef SIMDJSON_IMPLEMENTATION_ICELAKE

include/simdjson/internal/dom_parser_implementation.h

@@ -163,6 +163,7 @@ public:
    */
   simdjson_warn_unused inline error_code allocate(size_t capacity, size_t max_depth) noexcept;
 
+
 protected:
   /**
    * The maximum document length this parser supports.

include/simdjson/ppc64/implementation.h

@@ -26,6 +26,7 @@ public:
                                          size_t &dst_len) const noexcept final;
   simdjson_warn_unused bool validate_utf8(const char *buf,
                                           size_t len) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final;
 };
 
 } // namespace ppc64

include/simdjson/simdjson_version.h

@@ -4,14 +4,14 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION 1.0.2
+#define SIMDJSON_VERSION 2.0.4
 
 namespace simdjson {
 enum {
   /**
    * The major version (MAJOR.minor.revision) of simdjson being used.
    */
-  SIMDJSON_VERSION_MAJOR = 1,
+  SIMDJSON_VERSION_MAJOR = 2,
   /**
    * The minor version (major.MINOR.revision) of simdjson being used.
    */
@@ -19,7 +19,7 @@ enum {
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 2
+  SIMDJSON_VERSION_REVISION = 4
 };
 } // namespace simdjson
 

include/simdjson/westmere/implementation.h

@@ -22,6 +22,7 @@ public:
   ) const noexcept final;
   simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
   simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final;
 };
 
 } // namespace westmere

singleheader/amalgamate.py

@@ -9,6 +9,7 @@ import subprocess
 import os
 import re
 import shutil
+import datetime
 
 if sys.version_info < (3, 0):
     sys.stdout.write("Sorry, requires Python 3.x or better\n")
@@ -80,8 +81,10 @@ def dofile(fid, prepath, filename):
     # print(f"// dofile: invoked with prepath={prepath}, filename={filename}",file=fid)
     file = os.path.join(prepath, filename)
     RELFILE = os.path.relpath(file, PROJECTPATH)
+    # Windows use \ as a directory separator, but we do not want that:
+    OSRELFILE = RELFILE.replace('\\','/')
     # Last lines are always ignored. Files should end by an empty lines.
-    print(f"/* begin file {RELFILE} */", file=fid)
+    print(f"/* begin file {OSRELFILE} */", file=fid)
     includepattern = re.compile('^#include "(.*)"')
     redefines_simdjson_implementation = re.compile('^#define\s+SIMDJSON_IMPLEMENTATION\s+(.*)')
     undefines_simdjson_implementation = re.compile('^#undef\s+SIMDJSON_IMPLEMENTATION\s*$')
@@ -113,7 +116,7 @@ def dofile(fid, prepath, filename):
                 else:
                     # copy the line, with SIMDJSON_IMPLEMENTATION replace to what it is currently defined to
                     print(uses_simdjson_implementation.sub(current_implementation+"\\1",line), file=fid)
-    print(f"/* end file {RELFILE} */", file=fid)
+    print(f"/* end file {OSRELFILE} */", file=fid)
 
 
 # Get the generation date from git, so the output is reproducible.
@@ -121,8 +124,12 @@ def dofile(fid, prepath, filename):
 # does not change with locale and timezone at time of generation.
 # Forcing it to be UTC is difficult, because it needs to be portable
 # between gnu date and busybox date.
-timestamp = subprocess.run(['git', 'show', '-s', '--format=%ci', 'HEAD'],
+try:
+    timestamp = subprocess.run(['git', 'show', '-s', '--format=%ci', 'HEAD'],
                            stdout=subprocess.PIPE).stdout.decode('utf-8').strip()
+except:
+    print("git not found, timestamp based on current time")
+    timestamp = str(datetime.datetime.now())
 print(f"timestamp is {timestamp}")
 
 os.makedirs(AMALGAMATE_OUTPUT_PATH, exist_ok=True)

src/arm64/dom_parser_implementation.cpp

@@ -109,6 +109,7 @@ simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t>
 // Stage 2
 //
 
+#include "generic/stage2/stringparsing.h"
 #include "generic/stage2/tape_builder.h"
 
 //
@@ -151,6 +152,11 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
+simdjson_warn_unused uint8_t *implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return arm64::stringparsing::parse_string(src, dst);
+}
+
+
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {
   auto error = stage1(_buf, _len, stage1_mode::regular);
   if (error) { return error; }

src/fallback/dom_parser_implementation.cpp

@@ -346,6 +346,7 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t
 //
 // Stage 2
 //
+#include "generic/stage2/stringparsing.h"
 #include "generic/stage2/tape_builder.h"
 
 namespace simdjson {
@@ -359,6 +360,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
+simdjson_warn_unused uint8_t *implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return fallback::stringparsing::parse_string(src, dst);
+}
+
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {
   auto error = stage1(_buf, _len, stage1_mode::regular);
   if (error) { return error; }

src/generic/stage2/stringparsing.h

@@ -0,0 +1,142 @@
+// This file contains the common code every implementation uses
+// It is intended to be included multiple times and compiled multiple times
+
+namespace simdjson {
+namespace SIMDJSON_IMPLEMENTATION {
+namespace {
+/// @private
+namespace stringparsing {
+
+// begin copypasta
+// These chars yield themselves: " \ /
+// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab
+// u not handled in this table as it's complex
+static const uint8_t escape_map[256] = {
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0, // 0x0.
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+    0, 0, 0x22, 0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0x2f,
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0, // 0x4.
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0x5c, 0, 0,    0, // 0x5.
+    0, 0, 0x08, 0, 0,    0, 0x0c, 0, 0, 0, 0, 0, 0,    0, 0x0a, 0, // 0x6.
+    0, 0, 0x0d, 0, 0x09, 0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0, // 0x7.
+
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+    0, 0, 0,    0, 0,    0, 0,    0, 0, 0, 0, 0, 0,    0, 0,    0,
+};
+
+// handle a unicode codepoint
+// write appropriate values into dest
+// src will advance 6 bytes or 12 bytes
+// dest will advance a variable amount (return via pointer)
+// return true if the unicode codepoint was valid
+// We work in little-endian then swap at write time
+simdjson_warn_unused
+simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
+                                            uint8_t **dst_ptr) {
+  // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the
+  // conversion isn't valid; we defer the check for this to inside the
+  // multilingual plane check
+  uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
+  *src_ptr += 6;
+  // check for low surrogate for characters outside the Basic
+  // Multilingual Plane.
+  if (code_point >= 0xd800 && code_point < 0xdc00) {
+    if (((*src_ptr)[0] != '\\') || (*src_ptr)[1] != 'u') {
+      return false;
+    }
+    uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2);
+
+    // if the first code point is invalid we will get here, as we will go past
+    // the check for being outside the Basic Multilingual plane. If we don't
+    // find a \u immediately afterwards we fail out anyhow, but if we do,
+    // this check catches both the case of the first code point being invalid
+    // or the second code point being invalid.
+    if ((code_point | code_point_2) >> 16) {
+      return false;
+    }
+
+    code_point =
+        (((code_point - 0xd800) << 10) | (code_point_2 - 0xdc00)) + 0x10000;
+    *src_ptr += 6;
+  }
+  size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr);
+  *dst_ptr += offset;
+  return offset > 0;
+}
+
+/**
+ * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There
+ * must be an unescaped quote terminating the string. It returns the final output
+ * position as pointer. In case of error (e.g., the string has bad escaped codes),
+ * then null_nullptrptr is returned. It is assumed that the output buffer is large
+ * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes +
+ * SIMDJSON_PADDING bytes.
+ */
+simdjson_warn_unused simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) {
+  while (1) {
+    // Copy the next n bytes, and find the backslash and quote in them.
+    auto bs_quote = backslash_and_quote::copy_and_find(src, dst);
+    // If the next thing is the end quote, copy and return
+    if (bs_quote.has_quote_first()) {
+      // we encountered quotes first. Move dst to point to quotes and exit
+      return dst + bs_quote.quote_index();
+    }
+    if (bs_quote.has_backslash()) {
+      /* find out where the backspace is */
+      auto bs_dist = bs_quote.backslash_index();
+      uint8_t escape_char = src[bs_dist + 1];
+      /* we encountered backslash first. Handle backslash */
+      if (escape_char == 'u') {
+        /* move src/dst up to the start; they will be further adjusted
+           within the unicode codepoint handling code. */
+        src += bs_dist;
+        dst += bs_dist;
+        if (!handle_unicode_codepoint(&src, &dst)) {
+          return nullptr;
+        }
+      } else {
+        /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and
+         * write bs_dist+1 characters to output
+         * note this may reach beyond the part of the buffer we've actually
+         * seen. I think this is ok */
+        uint8_t escape_result = escape_map[escape_char];
+        if (escape_result == 0u) {
+          return nullptr; /* bogus escape value is an error */
+        }
+        dst[bs_dist] = escape_result;
+        src += bs_dist + 2;
+        dst += bs_dist + 1;
+      }
+    } else {
+      /* they are the same. Since they can't co-occur, it means we
+       * encountered neither. */
+      src += backslash_and_quote::BYTES_PROCESSED;
+      dst += backslash_and_quote::BYTES_PROCESSED;
+    }
+  }
+  /* can't be reached */
+  return nullptr;
+}
+
+simdjson_unused simdjson_warn_unused simdjson_really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *&current_string_buf_loc, std::string_view &s) {
+  if (*(src++) != '"') { return STRING_ERROR; }
+  auto end = stringparsing::parse_string(src, current_string_buf_loc);
+  if (!end) { return STRING_ERROR; }
+  s = std::string_view(reinterpret_cast<const char *>(current_string_buf_loc), end-current_string_buf_loc);
+  current_string_buf_loc = end;
+  return SUCCESS;
+}
+
+} // namespace stringparsing
+} // unnamed namespace
+} // namespace SIMDJSON_IMPLEMENTATION
+} // namespace simdjson

src/haswell/dom_parser_implementation.cpp

@@ -112,6 +112,7 @@ simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t>
 //
 // Stage 2
 //
+#include "generic/stage2/stringparsing.h"
 #include "generic/stage2/tape_builder.h"
 
 //
@@ -152,6 +153,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
+simdjson_warn_unused uint8_t *implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return haswell::stringparsing::parse_string(src, dst);
+}
+
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {
   auto error = stage1(_buf, _len, stage1_mode::regular);
   if (error) { return error; }

src/icelake/dom_parser_implementation.cpp

@@ -102,7 +102,7 @@ simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t>
 } // namespace SIMDJSON_IMPLEMENTATION
 } // namespace simdjson
 
-#include "icelake/utf8_lookup4_algorithm.h"
+#include "generic/stage1/utf8_lookup4_algorithm.h"
 // defining SIMDJSON_CUSTOM_BIT_INDEXER allows us to provide our own bit_indexer::write
 #define SIMDJSON_CUSTOM_BIT_INDEXER
 #include "generic/stage1/json_structural_indexer.h"
@@ -114,6 +114,9 @@ simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t>
  * naked intrinsics.
  * TODO: make this code more elegant.
  */
+// Under GCC 12, the intrinsic _mm512_extracti32x4_epi32 may generate 'maybe uninitialized'.
+// as a workaround, we disable warnings within the following function.
+SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
 namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { namespace { namespace stage1 {
 simdjson_really_inline void bit_indexer::write(uint32_t idx, uint64_t bits) {
     // In some instances, the next branch is expensive because it is mispredicted.
@@ -148,12 +151,14 @@ simdjson_really_inline void bit_indexer::write(uint32_t idx, uint64_t bits) {
     this->tail += count;
 }
 }}}}
+SIMDJSON_POP_DISABLE_WARNINGS
 
 #include "generic/stage1/utf8_validator.h"
 
 //
 // Stage 2
 //
+#include "generic/stage2/stringparsing.h"
 #include "generic/stage2/tape_builder.h"
 
 //
@@ -194,6 +199,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
+simdjson_warn_unused uint8_t *implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return icelake::stringparsing::parse_string(src, dst);
+}
+
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {
   auto error = stage1(_buf, _len, stage1_mode::regular);
   if (error) { return error; }

src/icelake/utf8_lookup4_algorithm.h

@@ -1,141 +0,0 @@
-namespace simdjson {
-namespace SIMDJSON_IMPLEMENTATION {
-namespace {
-namespace utf8_validation {
-
-simdjson_really_inline __m512i check_special_cases(__m512i input, const __m512i prev1) {
-  __m512i mask1 = _mm512_setr_epi64(
-        0x0202020202020202,
-        0x4915012180808080,
-        0x0202020202020202,
-        0x4915012180808080,
-        0x0202020202020202,
-        0x4915012180808080,
-        0x0202020202020202,
-        0x4915012180808080);
-
-    const __m512i v_0f = _mm512_set1_epi8(0x0f);
-    __m512i index1 = _mm512_and_si512(_mm512_srli_epi16(prev1, 4), v_0f);
-
-    __m512i byte_1_high = _mm512_shuffle_epi8(mask1, index1);
-    __m512i mask2 = _mm512_setr_epi64(
-        0xcbcbcb8b8383a3e7,
-        0xcbcbdbcbcbcbcbcb,
-        0xcbcbcb8b8383a3e7,
-        0xcbcbdbcbcbcbcbcb,
-        0xcbcbcb8b8383a3e7,
-        0xcbcbdbcbcbcbcbcb,
-        0xcbcbcb8b8383a3e7,
-        0xcbcbdbcbcbcbcbcb);
-     __m512i index2 = _mm512_and_si512(prev1, v_0f);
-
-    __m512i byte_1_low = _mm512_shuffle_epi8(mask2, index2);
-    __m512i mask3 = _mm512_setr_epi64(
-        0x101010101010101,
-        0x1010101babaaee6,
-        0x101010101010101,
-        0x1010101babaaee6,
-        0x101010101010101,
-        0x1010101babaaee6,
-        0x101010101010101,
-        0x1010101babaaee6
-    );
-    __m512i index3 = _mm512_and_si512(_mm512_srli_epi16(input, 4), v_0f);
-    __m512i byte_2_high = _mm512_shuffle_epi8(mask3, index3);
-    return _mm512_ternarylogic_epi64(byte_1_high, byte_1_low, byte_2_high, 128);
-  }
-
-  simdjson_really_inline __m512i check_multibyte_lengths(const __m512i prev2,
-      const __m512i prev3, const __m512i sc) {
-
-    __m512i is_third_byte  = _mm512_subs_epu8(prev2, _mm512_set1_epi8(0b11100000u-1)); // Only 111_____ will be > 0
-    __m512i is_fourth_byte  = _mm512_subs_epu8(prev3, _mm512_set1_epi8(0b11110000u-1)); // Only 1111____ will be > 0
-    __m512i is_third_or_fourth_byte = _mm512_or_si512(is_third_byte, is_fourth_byte);
-    const __m512i v_7f = _mm512_set1_epi8(char(0x7f));
-    is_third_or_fourth_byte = _mm512_adds_epu8(v_7f, is_third_or_fourth_byte);
-    // We want to compute (is_third_or_fourth_byte AND v80) XOR sc.
-    const __m512i v_80 = _mm512_set1_epi8(char(0x80));
-    return _mm512_ternarylogic_epi32(is_third_or_fourth_byte, v_80, sc, 0b1101010);
-    // We could also do it the long way:
-    //
-    //__m512i is_third_or_fourth_byte_mask = _mm512_and_si512(is_third_or_fourth_byte, v_80);
-    //return _mm512_xor_si512(is_third_or_fourth_byte_mask, sc);
-  }
-  //
-  // Return nonzero if there are incomplete multibyte characters at the end of the block:
-  // e.g. if there is a 4-byte character, but it's 3 bytes from the end.
-  //
-  simdjson_really_inline __m512i is_incomplete(const __m512i input) {
-    // If the previous input's last 3 bytes match this, they're too short (they ended at EOF):
-    // ... 1111____ 111_____ 11______
-    const __m512i max_value = _mm512_setr_epi64(
-        0xffffffffffffffff,
-        0xffffffffffffffff,
-        0xffffffffffffffff,
-        0xffffffffffffffff,
-        0xffffffffffffffff,
-        0xffffffffffffffff,
-        0xffffffffffffffff,
-        0xbfdfefffffffffff);
-    return _mm512_subs_epu8(input, max_value);
-  }
-
-  struct utf8_checker {
-    // If this is nonzero, there has been a UTF-8 error.
-    __m512i error{};
-
-    // The last input we received
-    __m512i prev_input_block{};
-    // Whether the last input we received was incomplete (used for ASCII fast path)
-    __m512i prev_incomplete{};
-
-    //
-    // Check whether the current bytes are valid UTF-8.
-    //
-    simdjson_really_inline void check_utf8_bytes(const __m512i input, const __m512i prev_input) {
-      // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes
-      // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers)
-      const __m512i movemask = _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6);
-      const __m512i rotated = _mm512_permutex2var_epi64(prev_input, movemask, input);
-      __m512i prev1 = _mm512_alignr_epi8(input, rotated, 16-1);
-      __m512i prev2 = _mm512_alignr_epi8(input, rotated, 16-2);
-      __m512i prev3 = _mm512_alignr_epi8(input, rotated, 16-3);
-      __m512i sc = check_special_cases(input, prev1);
-      this->error = _mm512_or_si512(check_multibyte_lengths(prev2, prev3, sc), this->error);
-    }
-
-    // The only problem that can happen at EOF is that a multibyte character is too short
-    // or a byte value too large in the last bytes: check_special_cases only checks for bytes
-    // too large in the first of two bytes.
-    simdjson_really_inline void check_eof() {
-      // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't
-      // possibly finish them.
-      this->error = _mm512_or_si512(this->error, this->prev_incomplete);
-    }
-
-    // returns true if ASCII.
-    simdjson_really_inline bool check_next_input(const __m512i input) {
-      const __m512i v_80 = _mm512_set1_epi8(char(0x80));
-      const __mmask64 ascii = _mm512_test_epi8_mask(input, v_80);
-      if(ascii == 0) {
-        this->error = _mm512_or_si512(this->error, this->prev_incomplete);
-        return true;
-      } else {
-        this->check_utf8_bytes(input, this->prev_input_block);
-        this->prev_incomplete = is_incomplete(input);
-        this->prev_input_block = input;
-        return false;
-      }
-    }
-    // do not forget to call check_eof!
-    simdjson_really_inline error_code errors() {
-      return (_mm512_test_epi8_mask(this->error, this->error) != 0) ? error_code::UTF8_ERROR : error_code::SUCCESS;
-    }
-  }; // struct utf8_checker
-} // namespace utf8_validation
-
-using utf8_validation::utf8_checker;
-
-} // unnamed namespace
-} // namespace SIMDJSON_IMPLEMENTATION
-} // namespace simdjson

src/implementation.cpp

@@ -72,6 +72,9 @@ public:
   simdjson_warn_unused bool validate_utf8(const char * buf, size_t len) const noexcept final override {
     return set_best()->validate_utf8(buf, len);
   }
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final override {
+    return set_best()->parse_string(src, dst);
+  }
   simdjson_really_inline detect_best_supported_implementation_on_first_use() noexcept : implementation("best_supported_detector", "Detects the best supported implementation and sets it", 0) {}
 private:
   const implementation *set_best() const noexcept;
@@ -124,6 +127,9 @@ public:
     // what are the chances that the programmer has a fallback? Given that *we* provide the
     // fallback, it implies that the programmer would need a fallback for our fallback.
   }
+  simdjson_warn_unused uint8_t *parse_string(const uint8_t *, uint8_t *) const noexcept final override {
+    return nullptr;
+  }
   unsupported_implementation() : implementation("unsupported", "Unsupported CPU (no detected SIMD instructions)", 0) {}
 };
 
@@ -188,7 +194,9 @@ simdjson_warn_unused error_code minify(const char *buf, size_t len, char *dst, s
 simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept {
   return get_active_implementation()->validate_utf8(buf, len);
 }
-
+simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) noexcept {
+  return get_active_implementation()->parse_string(src, dst);
+}
 const implementation * builtin_implementation() {
   static const implementation * builtin_impl = get_available_implementations()[SIMDJSON_STRINGIFY(SIMDJSON_BUILTIN_IMPLEMENTATION)];
   assert(builtin_impl);

src/ppc64/dom_parser_implementation.cpp

@@ -79,7 +79,7 @@ simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t>
 //
 // Stage 2
 //
-
+#include "generic/stage2/stringparsing.h"
 #include "generic/stage2/tape_builder.h"
 
 //
@@ -122,6 +122,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
+simdjson_warn_unused uint8_t *implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return ppc64::stringparsing::parse_string(src, dst);
+}
+
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {
   auto error = stage1(_buf, _len, stage1_mode::regular);
   if (error) { return error; }

src/westmere/dom_parser_implementation.cpp

@@ -110,6 +110,7 @@ simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t>
 //
 // Stage 2
 //
+#include "generic/stage2/stringparsing.h"
 #include "generic/stage2/tape_builder.h"
 
 //
@@ -151,6 +152,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu
   return stage2::tape_builder::parse_document<true>(*this, _doc);
 }
 
+simdjson_warn_unused uint8_t *implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept {
+  return westmere::stringparsing::parse_string(src, dst);
+}
+
 simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {
   auto error = stage1(_buf, _len, stage1_mode::regular);
   if (error) { return error; }

tests/ondemand/ondemand_array_error_tests.cpp

@@ -16,7 +16,7 @@ namespace array_error_tests {
     size_t count = 0;
     for (auto elem : std::forward<T>(array)) {
       std::cout << "-"; std::cout.flush();
-      V actual;
+      V actual{};
       auto actual_error = elem.get(actual);
       if (count >= N) {
         if (count >= (N+N2)) {

tests/ondemand/ondemand_assert_out_of_order_values.cpp

@@ -5,7 +5,14 @@
 #include <sys/types.h>
 #include "simdjson.h"
 
+
+// We get spurious "maybe used uninitialized" warnings under GCC 12.
 using namespace simdjson;
+#if defined(__GNUC__) && !defined(__clang__)
+#if __GNUC__ >= 12
+SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
+#endif
+#endif
 
 // This ensures the compiler can't rearrange them into the proper order (which causes it to work!)
 simdjson_never_inline bool check_point(simdjson_result<ondemand::value> xval, simdjson_result<ondemand::value> yval) {

tests/ondemand/ondemand_error_tests.cpp

@@ -6,6 +6,26 @@ using namespace simdjson;
 namespace error_tests {
   using namespace std;
 
+  bool issue1834() {
+    TEST_START();
+    ondemand::parser parser;
+    auto json = "[[]"_padded;
+    json.data()[json.size()] = ']';
+    auto doc = parser.iterate(json);
+    size_t cnt{};
+    auto error = doc.count_elements().get(cnt);
+    return error != simdjson::SUCCESS;
+  }
+  bool issue1834_2() {
+    TEST_START();
+    ondemand::parser parser;
+    auto json = "{\"a\":{}"_padded;
+    json.data()[json.size()] = '}';
+    auto doc = parser.iterate(json);
+    size_t cnt{};
+    auto error = doc.count_fields().get(cnt);
+    return error != simdjson::SUCCESS;
+  }
   bool empty_document_error() {
     TEST_START();
     ondemand::parser parser;
@@ -272,6 +292,8 @@ namespace error_tests {
 
   bool run() {
     return
+           issue1834() &&
+           issue1834_2() &&
 #if SIMDJSON_EXCEPTIONS
            raw_json_string_except() &&
            raw_json_string_except_with_io() &&

tests/ondemand/ondemand_number_tests.cpp

@@ -197,16 +197,18 @@ namespace number_tests {
   bool get_number_tests() {
     TEST_START();
     ondemand::parser parser;
-    padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999])"_padded;
+    padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999,-9223372036854775807,-9223372036854775808])"_padded;
     ondemand::number_type expectedtypes[] = {ondemand::number_type::floating_point_number,
                                  ondemand::number_type::signed_integer,
                                  ondemand::number_type::signed_integer,
                                  ondemand::number_type::floating_point_number,
                                  ondemand::number_type::signed_integer,
-                                 ondemand::number_type::unsigned_integer
+                                 ondemand::number_type::unsigned_integer,
+                                 ondemand::number_type::signed_integer,
+                                 ondemand::number_type::signed_integer
                                  };
-    bool is_negative[] = {false, false, false, false, true, false};
-    bool is_integer[] = {false, true, true, false, true, true};
+    bool is_negative[] = {false, false, false, false, true, false, true, true};
+    bool is_integer[] = {false, true, true, false, true, true, true, true};
 
     ondemand::document doc;
     ASSERT_SUCCESS(parser.iterate(docdata).get(doc));

tests/ondemand/ondemand_readme_examples.cpp

@@ -109,6 +109,85 @@ bool basics_treewalk() {
   return true;
 }
 
+void print_depth_space(ondemand::value element) {
+  for(auto i = 0; i < element.current_depth(); i++) {
+    cout << " ";
+  }
+}
+
+void recursive_print_json_breakline(ondemand::value element) {
+    bool add_comma;
+    switch (element.type()) {
+    case ondemand::json_type::array:
+      cout << endl;
+      print_depth_space(element);
+      cout << "[";
+      add_comma = false;
+      for (auto child : element.get_array()) {
+        if (add_comma) {
+          print_depth_space(element);
+          cout << ",";
+        }
+        // We need the call to value() to get
+        // an ondemand::value type.
+        recursive_print_json_breakline(child.value());
+        add_comma = true;
+      }
+      cout << "]";
+      break;
+    case ondemand::json_type::object:
+      cout << endl;
+      print_depth_space(element);
+      cout << "{";
+      add_comma = false;
+      for (auto field : element.get_object()) {
+        if (add_comma) {
+          print_depth_space(element);
+          cout << ",";
+        }
+        // key() returns the key as it appears in the raw
+        // JSON document, if we want the unescaped key,
+        // we should do field.unescaped_key().
+        cout << "\"" << field.key() << "\": ";
+        recursive_print_json_breakline(field.value());
+        add_comma = true;
+      }
+      cout << "}\n";
+      break;
+    case ondemand::json_type::number:
+      // assume it fits in a double
+      cout << element.get_double();
+      break;
+    case ondemand::json_type::string:
+      // get_string() would return escaped string, but
+      // we are happy with unescaped string.
+      cout << "\"" << element.get_raw_json_string() << "\"";
+      break;
+    case ondemand::json_type::boolean:
+      cout << element.get_bool();
+      break;
+    case ondemand::json_type::null:
+      cout << "null";
+      break;
+    }
+}
+
+bool basics_treewalk_breakline() {
+  padded_string json[3] = {R"( [
+    { "make": "Toyota", "model": "Camry",  "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
+    { "make": "Kia",    "model": "Soul",   "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
+    { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
+  ] )"_padded, R"( {"key":"value"} )"_padded, "[12,3]"_padded};
+  ondemand::parser parser;
+  for(size_t i = 0 ; i < 3; i++) {
+    ondemand::document doc = parser.iterate(json[i]);
+    ondemand::value val = doc;
+    recursive_print_json_breakline(val);
+    std::cout << std::endl;
+  }
+  return true;
+}
+
 bool basics_1() {
   TEST_START();
 
@@ -673,7 +752,7 @@ bool stream_capacity_example() {
   if( error ) { /* handle the error */ }
   for (auto doc: stream) {
     if(counter < 6) {
-      int64_t val;
+      int64_t val{};
       error = doc.at_pointer("/4").get(val);
       if( error ) { /* handle the error */ }
       std::cout << "5 = " << val << std::endl;
@@ -882,6 +961,7 @@ bool current_location_no_error() {
 int main() {
 #if SIMDJSON_EXCEPTIONS
   basics_treewalk();
+  basics_treewalk_breakline();
 #endif
   if (
     true

tools/minify.cpp

@@ -1,3 +1,4 @@
+#include <chrono>
 #include <iostream>
 #if (!(_MSC_VER) && !(__MINGW32__) && !(__MINGW64__))
 #include <dirent.h>
@@ -38,6 +39,7 @@ int main(int argc, const char *argv[]) {
   }
   options.add_options()
     ("a,arch", ss.str(), cxxopts::value<std::string>())
+    ("t,timing", "Report only timing.")
     ("f,file", "File name.", cxxopts::value<std::string>())
     ("h,help", "Print usage.")
   ;
@@ -79,7 +81,34 @@ int main(int argc, const char *argv[]) {
   size_t copy_len;
   error = simdjson::get_active_implementation()->minify((const uint8_t*)p.data(), p.length(), (uint8_t*)copy.data(), copy_len);
   if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
-  printf("%s", copy.data());
+  /**
+   * If a user only wants to time the required time, we do not output
+   * the result and we simply do the processing in a tight loop.
+   * At this point in time, we can assume that the processing will
+   * succeed.
+   */
+  if(result.count("timing")) {
+      uint64_t beforens = std::chrono::duration_cast<::std::chrono::nanoseconds>(
+             std::chrono::steady_clock::now().time_since_epoch())
+             .count();
+      error = simdjson::get_active_implementation()->minify((const uint8_t*)p.data(), p.length(), (uint8_t*)copy.data(), copy_len);
+      uint64_t afterns = std::chrono::duration_cast<::std::chrono::nanoseconds>(
+             std::chrono::steady_clock::now().time_since_epoch())
+             .count();
+      size_t times = 1;
+      while(afterns - beforens < 1000000000) {
+          error = simdjson::get_active_implementation()->minify((const uint8_t*)p.data(), p.length(), (uint8_t*)copy.data(), copy_len);
+          afterns = std::chrono::duration_cast<::std::chrono::nanoseconds>(
+             std::chrono::steady_clock::now().time_since_epoch())
+             .count();
+          times += 1;
+      }
+      if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
+      printf("%.3f GB/s\n", double(p.length() * times) / double(afterns - beforens));
+  } else {
+      // This is the expected path:
+      printf("%s", copy.data());
+  }
   return EXIT_SUCCESS;
 #ifdef __cpp_exceptions
   } catch (const cxxopts::OptionException& e) {