From 6029af75b34cc1719358733f675416d8c1cd3b83 Mon Sep 17 00:00:00 2001
From: evbse <133204505+evbse@users.noreply.github.com>
Date: Tue, 15 Jul 2025 21:39:09 +0300
Subject: [PATCH] Improve fallback implementation (#2393)

---
 src/fallback.cpp | 120 ++++++++++++++++++++++++++++++++++-------------
 1 file changed, 88 insertions(+), 32 deletions(-)

diff --git a/src/fallback.cpp b/src/fallback.cpp
index 46881a228..0f8865de0 100644
--- a/src/fallback.cpp
+++ b/src/fallback.cpp
@@ -121,10 +121,78 @@ simdjson_inline void validate_utf8_character() {
   idx += 4;
 }
 
+static const uint8_t CHAR_TYPE_SPACE     = 1 << 0;
+static const uint8_t CHAR_TYPE_OPERATOR  = 1 << 1;
+static const uint8_t CHAR_TYPE_ESC_ASCII = 1 << 2;
+static const uint8_t CHAR_TYPE_NON_ASCII = 1 << 3;
+
+const uint8_t char_table[256] = {
+  0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04,
+  0x04, 0x05, 0x05, 0x04, 0x04, 0x05, 0x04, 0x04,
+  0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04,
+  0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04,
+  0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x02, 0x04, 0x02, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x02, 0x00, 0x02, 0x00, 0x00,
+  0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+  0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+  0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+  0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+  0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+  0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+  0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+  0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+  0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+  0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+  0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+  0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+  0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+  0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+  0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
+  0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08
+};
+
+simdjson_inline bool char_is_type(uint8_t c, uint8_t type) {
+  return (char_table[c] & type);
+}
+
+simdjson_inline bool char_is_space(uint8_t c) {
+  return char_is_type(c, CHAR_TYPE_SPACE);
+}
+
+simdjson_inline bool char_is_operator(uint8_t c) {
+  return char_is_type(c, CHAR_TYPE_OPERATOR);
+}
+
+simdjson_inline bool char_is_space_or_operator(uint8_t c) {
+  return char_is_type(c, CHAR_TYPE_SPACE | CHAR_TYPE_OPERATOR);
+}
+
+simdjson_inline bool char_is_ascii_stop(uint8_t c) {
+  return char_is_type(c, CHAR_TYPE_ESC_ASCII | CHAR_TYPE_NON_ASCII);
+}
+
 // Returns true if the string is unclosed.
 simdjson_inline bool validate_string() {
   idx++; // skip first quote
-  while (idx < len && buf[idx] != '"') {
+  while (idx < len) {
+    do {
+      if (char_is_ascii_stop(buf[idx])) { break; }
+      idx++;
+    } while (idx < len);
+    if (idx >= len) { return true; }
+    if (buf[idx] == '"') {
+      return false;
+    }
     if (buf[idx] == '\\') {
       idx += 2;
     } else if (simdjson_unlikely(buf[idx] & 0x80)) {
@@ -138,43 +206,31 @@ simdjson_inline bool validate_string() {
   return false;
 }
 
-simdjson_inline bool is_whitespace_or_operator(uint8_t c) {
-  switch (c) {
-    case '{': case '}': case '[': case ']': case ',': case ':':
-    case ' ': case '\r': case '\n': case '\t':
-      return true;
-    default:
-      return false;
-  }
-}
-
 //
 // Parse the entire input in STEP_SIZE-byte chunks.
 //
 simdjson_inline error_code scan() {
   bool unclosed_string = false;
   for (;idx<len;idx++) {
-    switch (buf[idx]) {
-      // String
-      case '"':
-        add_structural();
-        unclosed_string |= validate_string();
-        break;
-      // Operator
-      case '{': case '}': case '[': case ']': case ',': case ':':
-        add_structural();
-        break;
-      // Whitespace
-      case ' ': case '\r': case '\n': case '\t':
-        break;
-      // Primitive or invalid character (invalid characters will be checked in stage 2)
-      default:
-        // Anything else, add the structural and go until we find the next one
-        add_structural();
-        while (idx+1<len && !is_whitespace_or_operator(buf[idx+1])) {
-          idx++;
-        };
-        break;
+    do {
+      if (!char_is_space(buf[idx])) { break; }
+      idx++;
+    } while (idx < len);
+    if (idx >= len) { break; }
+    // String
+    if (buf[idx] == '"') {
+      add_structural();
+      unclosed_string |= validate_string();
+    // Operator
+    } else if (char_is_operator(buf[idx])) {
+      add_structural();
+    // Primitive or invalid character (invalid characters will be checked in stage 2)
+    } else {
+      // Anything else, add the structural and go until we find the next one
+      add_structural();
+      while (idx+1<len && !char_is_space_or_operator(buf[idx+1])) {
+        idx++;
+      };
     }
   }
   // We pad beyond.