From 714ef4c4fdd4d09999d61d4b0d9b7653f1a1188d Mon Sep 17 00:00:00 2001
From: "Yukihiro \"Matz\" Matsumoto" <matz@ruby.or.jp>
Date: Mon, 29 Apr 2024 15:37:22 +0900
Subject: [PATCH] string.c (mrb_utf8_strlen): handle invalid UTF-8 sequence;
 fix #6255

Previous SWAR version assumes valid UTF-8 to count number of code points
in the string, but we need to handle invalid sequence as well. We now
use `search_nonascii` to skip counting single byte characters for
performance. The new version is even faster than SWAR version (probably
because `search_nonascii` uses SSE2 on Intel compatible CPU (which I use).
---
 src/string.c | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/src/string.c b/src/string.c
index 46a0dc847..40f7081b0 100644
--- a/src/string.c
+++ b/src/string.c
@@ -417,26 +417,19 @@ static inline uint32_t popcount(bitint x)
 mrb_int
 mrb_utf8_strlen(const char *str, mrb_int byte_len)
 {
+  const char *p = str;
+  const char *e = str + byte_len;
   mrb_int len = 0;
 
-  const char *p = str;
-  const char *be = p + sizeof(bitint) * (byte_len / sizeof(bitint));
-  for (; p < be; p+=sizeof(bitint)) {
-    bitint t0;
+  while (p < e) {
+    const char *np = search_nonascii(p, e);
 
-    memcpy(&t0, p, sizeof(bitint));
-    const bitint t1 = t0 & (MASK1*0xc0);
-    const bitint t2 = t1 + (MASK1*0x40);
-    const bitint t3 = t1 & t2;
-    len += popcount(t3);
-  }
-  len = sizeof(bitint) * (byte_len / sizeof(bitint)) - len;
-
-  if (byte_len % sizeof(bitint)) {
-    const char *e = str + byte_len;
-    while (p < e) {
-      if (utf8_islead(*p)) len++;
-      p++;
+    len += np - p;
+    if (np == e) break;
+    p = np;
+    while (NOASCII(*p)) {
+      p += mrb_utf8len(p, e);
+      len++;
     }
   }
   return len;