admin/mruby-mruby
1
0
Fork 0
mirror of https://github.com/mruby/mruby synced 2026-06-08 16:11:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

string.c: add mrb_utf8_to_buf() to consolidate UTF-8 encoding

Browse Source 
Extract duplicated UTF-8 codepoint-to-bytes encoding into a shared
function in src/string.c. Update all gems to use it:

- mruby-sprintf: %c specifier
- mruby-io: putc
- mruby-string-ext: Integer#chr
- mruby-pack: pack("U")
- mruby-compiler: Unicode escapes in parser

Also use existing mrb_utf8len() in io.c for character length detection.

Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Yukihiro "Matz" Matsumoto
2025-12-18 16:30:03 +09:00
parent 53fce124e6
commit 7e28e68dca
8 changed files with 50 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/mruby/internal.h
+1
View File
@@ -166,6 +166,7 @@ mrb_value mrb_str_aref(mrb_state *mrb, mrb_value str, mrb_value idx, mrb_value l
uint32_t mrb_byte_hash(const uint8_t*, mrb_int);
uint32_t mrb_byte_hash_step(const uint8_t*, mrb_int, uint32_t);
mrb_int mrb_utf8_to_buf(char *buf, uint32_t cp);
#ifdef MRB_UTF8_STRING
mrb_int mrb_utf8len(const char *str, const char *end);
mrb_int mrb_utf8_strlen(const char *str, mrb_int byte_len);
mrbgems/mruby-compiler/core/parse.y
+2 -24
View File
@@ -5375,30 +5375,8 @@ tokadd(parser_state *p, int32_t c)
len = 1;
}
else {
/* Unicode character */
c = -c;
if (c < 0x80) {
utf8[0] = (char)c;
len = 1;
}
else if (c < 0x800) {
utf8[0] = (char)(0xC0 | (c >> 6));
utf8[1] = (char)(0x80 | (c & 0x3F));
len = 2;
}
else if (c < 0x10000) {
utf8[0] = (char)(0xE0 | (c >> 12) );
utf8[1] = (char)(0x80 | ((c >> 6) & 0x3F));
utf8[2] = (char)(0x80 | ( c & 0x3F));
len = 3;
}
else {
utf8[0] = (char)(0xF0 | (c >> 18) );
utf8[1] = (char)(0x80 | ((c >> 12) & 0x3F));
utf8[2] = (char)(0x80 | ((c >> 6) & 0x3F));
utf8[3] = (char)(0x80 | ( c & 0x3F));
len = 4;
}
/* Unicode character (negative c indicates codepoint) */
len = (int)mrb_utf8_to_buf(utf8, (uint32_t)(-c));
}
if (p->tidx+len >= p->tsiz) {
if (p->tsiz >= MRB_PARSER_TOKBUF_MAX) {
mrbgems/mruby-compiler/core/y.tab.c
+2 -24
View File
@@ -12613,30 +12613,8 @@ tokadd(parser_state *p, int32_t c)
len = 1;
}
else {
/* Unicode character */
c = -c;
if (c < 0x80) {
utf8[0] = (char)c;
len = 1;
}
else if (c < 0x800) {
utf8[0] = (char)(0xC0 | (c >> 6));
utf8[1] = (char)(0x80 | (c & 0x3F));
len = 2;
}
else if (c < 0x10000) {
utf8[0] = (char)(0xE0 | (c >> 12) );
utf8[1] = (char)(0x80 | ((c >> 6) & 0x3F));
utf8[2] = (char)(0x80 | ( c & 0x3F));
len = 3;
}
else {
utf8[0] = (char)(0xF0 | (c >> 18) );
utf8[1] = (char)(0x80 | ((c >> 12) & 0x3F));
utf8[2] = (char)(0x80 | ((c >> 6) & 0x3F));
utf8[3] = (char)(0x80 | ( c & 0x3F));
len = 4;
}
/* Unicode character (negative c indicates codepoint) */
len = (int)mrb_utf8_to_buf(utf8, (uint32_t)(-c));
}
if (p->tidx+len >= p->tsiz) {
if (p->tsiz >= MRB_PARSER_TOKBUF_MAX) {
mrbgems/mruby-io/src/io.c
+1 -18
View File
@@ -1124,24 +1124,7 @@ io_putc(mrb_state *mrb, mrb_value io)
if (len == 0) return c;
#ifdef MRB_UTF8_STRING
/* Determine UTF-8 character length from first byte */
unsigned char first = (unsigned char)ptr[0];
if (first < 0x80) {
write_len = 1; /* ASCII */
}
else if ((first & 0xE0) == 0xC0) {
write_len = 2; /* 2-byte UTF-8 */
}
else if ((first & 0xF0) == 0xE0) {
write_len = 3; /* 3-byte UTF-8 */
}
else if ((first & 0xF8) == 0xF0) {
write_len = 4; /* 4-byte UTF-8 */
}
else {
write_len = 1; /* Invalid UTF-8, write single byte */
}
if (write_len > len) write_len = len;
write_len = mrb_utf8len(ptr, ptr + len);
#else
write_len = 1; /* Non-UTF8: write single byte */
#endif
mrbgems/mruby-pack/src/pack.c
+5 -29
View File
@@ -10,6 +10,7 @@
#include <mruby/string.h>
#include <mruby/variable.h>
#include <mruby/endian.h>
#include <mruby/internal.h>
#include <mruby/presym.h>
#include <ctype.h>
@@ -772,36 +773,11 @@ static int
pack_utf8(mrb_state *mrb, mrb_value o, mrb_value str, mrb_int sidx, int count, unsigned int flags)
{
char utf8[4];
int len = 0;
uint32_t c = 0;
int len;
uint32_t c = (uint32_t)mrb_integer(o);
c = (uint32_t)mrb_integer(o);
/* Unicode character */
/* from mruby-compiler gem */
if (c < 0x80) {
utf8[0] = (char)c;
len = 1;
}
else if (c < 0x800) {
utf8[0] = (char)(0xC0 | (c >> 6));
utf8[1] = (char)(0x80 | (c & 0x3F));
len = 2;
}
else if (c < 0x10000) {
utf8[0] = (char)(0xE0 | (c >> 12) );
utf8[1] = (char)(0x80 | ((c >> 6) & 0x3F));
utf8[2] = (char)(0x80 | ( c & 0x3F));
len = 3;
}
else if (c < 0x200000) {
utf8[0] = (char)(0xF0 | (c >> 18) );
utf8[1] = (char)(0x80 | ((c >> 12) & 0x3F));
utf8[2] = (char)(0x80 | ((c >> 6) & 0x3F));
utf8[3] = (char)(0x80 | ( c & 0x3F));
len = 4;
}
else {
len = (int)mrb_utf8_to_buf(utf8, c);
if (len == 0) {
mrb_raise(mrb, E_RANGE_ERROR, "pack(U): value out of range");
}
mrbgems/mruby-sprintf/src/sprintf.c
+2 -22
View File
@@ -520,28 +520,8 @@ retry:
/* Integer: encode directly to stack buffer (no allocation) */
mrb_int code = mrb_integer(val);
#ifdef MRB_UTF8_STRING
if (code < 0x80) {
cbuf[0] = (char)code;
clen = 1;
}
else if (code < 0x800) {
cbuf[0] = (char)(0xC0 | (code >> 6));
cbuf[1] = (char)(0x80 | (code & 0x3F));
clen = 2;
}
else if (code < 0x10000) {
cbuf[0] = (char)(0xE0 | (code >> 12));
cbuf[1] = (char)(0x80 | ((code >> 6) & 0x3F));
cbuf[2] = (char)(0x80 | (code & 0x3F));
clen = 3;
}
else {
cbuf[0] = (char)(0xF0 | (code >> 18));
cbuf[1] = (char)(0x80 | ((code >> 12) & 0x3F));
cbuf[2] = (char)(0x80 | ((code >> 6) & 0x3F));
cbuf[3] = (char)(0x80 | (code & 0x3F));
clen = 4;
}
clen = (int)mrb_utf8_to_buf(cbuf, (uint32_t)code);
if (clen == 0) clen = 1; /* invalid codepoint: write single byte */
#else
cbuf[0] = (char)(code & 0xff);
clen = 1;
mrbgems/mruby-string-ext/src/string.c
+4 -25
View File
@@ -51,36 +51,15 @@ int_chr_utf8(mrb_state *mrb, mrb_value num)
char utf8[4];
mrb_int len;
mrb_value str;
uint32_t sb_flag = 0;
if (cp < 0 || 0x10FFFF < cp) {
mrb_raisef(mrb, E_RANGE_ERROR, "%v out of char range", num);
}
if (cp < 0x80) {
utf8[0] = (char)cp;
len = 1;
sb_flag = MRB_STR_SINGLE_BYTE;
}
else if (cp < 0x800) {
utf8[0] = (char)(0xC0 | (cp >> 6));
utf8[1] = (char)(0x80 | (cp & 0x3F));
len = 2;
}
else if (cp < 0x10000) {
utf8[0] = (char)(0xE0 | (cp >> 12));
utf8[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
utf8[2] = (char)(0x80 | ( cp & 0x3F));
len = 3;
}
else {
utf8[0] = (char)(0xF0 | (cp >> 18));
utf8[1] = (char)(0x80 | ((cp >> 12) & 0x3F));
utf8[2] = (char)(0x80 | ((cp >> 6) & 0x3F));
utf8[3] = (char)(0x80 | ( cp & 0x3F));
len = 4;
}
len = mrb_utf8_to_buf(utf8, (uint32_t)cp);
str = mrb_str_new(mrb, utf8, len);
mrb_str_ptr(str)->flags |= sb_flag;
if (len == 1) {
RSTR_SET_ASCII_FLAG(mrb_str_ptr(str));
}
return str;
}
#endif
src/string.c
+33
View File
@@ -316,6 +316,39 @@ mrb_gc_free_str(mrb_state *mrb, struct RString *str)
#define MASK01 0x01010101ul
#endif
/*
* Encode a Unicode codepoint to UTF-8 bytes.
* buf must have at least 4 bytes of space.
* Returns the number of bytes written (1-4), or 0 for invalid codepoint.
*/
mrb_int
mrb_utf8_to_buf(char *buf, uint32_t cp)
{
if (cp < 0x80) {
buf[0] = (char)cp;
return 1;
}
else if (cp < 0x800) {
buf[0] = (char)(0xC0 | (cp >> 6));
buf[1] = (char)(0x80 | (cp & 0x3F));
return 2;
}
else if (cp < 0x10000) {
buf[0] = (char)(0xE0 | (cp >> 12));
buf[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
buf[2] = (char)(0x80 | (cp & 0x3F));
return 3;
}
else if (cp <= 0x10FFFF) {
buf[0] = (char)(0xF0 | (cp >> 18));
buf[1] = (char)(0x80 | ((cp >> 12) & 0x3F));
buf[2] = (char)(0x80 | ((cp >> 6) & 0x3F));
buf[3] = (char)(0x80 | (cp & 0x3F));
return 4;
}
return 0; /* invalid codepoint */
}
#ifdef MRB_UTF8_STRING
#define NOASCII(c) ((c) & 0x80)