mruby-string-ext: reject surrogates and ill-formed UTF-8

Make `Integer#chr("UTF-8")` reject UTF-16 surrogate code points
(U+D800..U+DFFF), and make `String#ord` reject ill-formed UTF-8 byte
sequences (overlong encodings, surrogates encoded as UTF-8, and code
points above U+10FFFF), matching CRuby and RFC 3629.

The `utf8code()` helper now decodes the code point first and then
validates the range per byte length:
  len=2: cp >= 0x80                 (rejects overlong)
  len=3: cp >= 0x800 and not D800..DFFF
  len=4: 0x10000 <= cp <= 0x10FFFF

close #2708

Co-authored-by: Claude <noreply@anthropic.com>

mrbgems/mruby-string-ext/src/string.c

@@ -33,7 +33,8 @@ int_chr_utf8(mrb_state *mrb, mrb_value num)
   mrb_int len;
   mrb_value str;
 
-  if (cp < 0 || 0x10FFFF < cp) {
+  /* Reject negative, above U+10FFFF, and UTF-16 surrogates (RFC 3629). */
+  if (cp < 0 || 0x10FFFF < cp || (0xD800 <= cp && cp <= 0xDFFF)) {
     mrb_raisef(mrb, E_RANGE_ERROR, "%v out of char range", num);
   }
   len = mrb_utf8_to_buf(utf8, (uint32_t)cp);
@@ -973,20 +974,27 @@ utf8code(mrb_state* mrb, const unsigned char* p, const unsigned char *e)
   if (p[0] < 0x80) return p[0];
 
   mrb_int len = mrb_utf8len_table[p[0]>>3];
+  mrb_int cp = -1;
   if (p+len <= e && len > 1 && (p[1] & 0xc0) == 0x80) {
     if (len == 2)
-      return ((p[0] & 0x1f) << 6) + (p[1] & 0x3f);
-    if ((p[2] & 0xc0) == 0x80) {
+      cp = ((p[0] & 0x1f) << 6) + (p[1] & 0x3f);
+    else if ((p[2] & 0xc0) == 0x80) {
       if (len == 3)
-        return ((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6)
-          + (p[2] & 0x3f);
-      if ((p[3] & 0xc0) == 0x80) {
-        if (len == 4)
-          return ((p[0] & 0x07) << 18) + ((p[1] & 0x3f) << 12)
-            + ((p[2] & 0x3f) << 6) + (p[3] & 0x3f);
+        cp = ((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6) + (p[2] & 0x3f);
+      else if (len == 4 && (p[3] & 0xc0) == 0x80) {
+        cp = ((p[0] & 0x07) << 18) + ((p[1] & 0x3f) << 12)
+              + ((p[2] & 0x3f) << 6) + (p[3] & 0x3f);
       }
     }
   }
+  /* Reject overlong sequences, UTF-16 surrogates, and code points above
+     U+10FFFF (RFC 3629, Unicode D93b). */
+  if (cp >= 0 &&
+      ((len == 2 && cp >= 0x80) ||
+       (len == 3 && cp >= 0x800 && (cp < 0xD800 || 0xDFFF < cp)) ||
+       (len == 4 && cp >= 0x10000 && cp <= 0x10FFFF))) {
+    return cp;
+  }
   mrb_raise(mrb, E_ARGUMENT_ERROR, "invalid UTF-8 byte sequence");
   /* not reached */
   return -1;

mrbgems/mruby-string-ext/test/numeric.rb

@@ -22,6 +22,11 @@ assert('Integer#chr') do
     assert_equal("«", 171.chr("utf-8"))
     assert_equal("あ", 12354.chr("Utf-8"))
     assert_raise(RangeError) { -1.chr("utf-8") }
-    assert_raise(RangeError) { 0x110000.chr.chr("UTF-8") }
+    assert_raise(RangeError) { 0x110000.chr("UTF-8") }
+    # UTF-16 surrogates are not valid Unicode scalar values (RFC 3629, #2708)
+    assert_raise(RangeError) { 0xD800.chr("UTF-8") }
+    assert_raise(RangeError) { 0xDFFF.chr("UTF-8") }
+    assert_equal "\u{D7FF}", 0xD7FF.chr("UTF-8")
+    assert_equal "\u{E000}", 0xE000.chr("UTF-8")
   end
 end

mrbgems/mruby-string-ext/test/string.rb

@@ -667,6 +667,19 @@ assert('String#ord(UTF-8)') do
   assert_equal expect, got
 end if UTF8STRING
 
+assert('String#ord(UTF-8) rejects ill-formed sequences', '#2708') do
+  # overlong encodings (RFC 3629)
+  assert_raise(ArgumentError) { "\xC0\x80".ord }       # 2-byte overlong NUL
+  assert_raise(ArgumentError) { "\xE0\x80\x80".ord }   # 3-byte overlong NUL
+  assert_raise(ArgumentError) { "\xF0\x80\x80\x80".ord } # 4-byte overlong NUL
+  assert_raise(ArgumentError) { "\xE0\x9F\xBF".ord }   # overlong U+07FF as 3 bytes
+  # UTF-16 surrogates encoded as UTF-8
+  assert_raise(ArgumentError) { "\xED\xA0\x80".ord }   # U+D800
+  assert_raise(ArgumentError) { "\xED\xBF\xBF".ord }   # U+DFFF
+  # above U+10FFFF
+  assert_raise(ArgumentError) { "\xF4\x90\x80\x80".ord } # U+110000
+end if UTF8STRING
+
 assert('String#chr') do
   assert_equal "a", "abcde".chr
   assert_equal "h", "hello!".chr