diff --git a/include/mruby/internal.h b/include/mruby/internal.h index 3ec83fa3a..89cd5d52b 100644 --- a/include/mruby/internal.h +++ b/include/mruby/internal.h @@ -166,6 +166,7 @@ mrb_value mrb_str_aref(mrb_state *mrb, mrb_value str, mrb_value idx, mrb_value l uint32_t mrb_byte_hash(const uint8_t*, mrb_int); uint32_t mrb_byte_hash_step(const uint8_t*, mrb_int, uint32_t); +mrb_int mrb_utf8_to_buf(char *buf, uint32_t cp); #ifdef MRB_UTF8_STRING mrb_int mrb_utf8len(const char *str, const char *end); mrb_int mrb_utf8_strlen(const char *str, mrb_int byte_len); diff --git a/mrbgems/mruby-compiler/core/parse.y b/mrbgems/mruby-compiler/core/parse.y index 383b23b54..7e5a2a0b8 100644 --- a/mrbgems/mruby-compiler/core/parse.y +++ b/mrbgems/mruby-compiler/core/parse.y @@ -5375,30 +5375,8 @@ tokadd(parser_state *p, int32_t c) len = 1; } else { - /* Unicode character */ - c = -c; - if (c < 0x80) { - utf8[0] = (char)c; - len = 1; - } - else if (c < 0x800) { - utf8[0] = (char)(0xC0 | (c >> 6)); - utf8[1] = (char)(0x80 | (c & 0x3F)); - len = 2; - } - else if (c < 0x10000) { - utf8[0] = (char)(0xE0 | (c >> 12) ); - utf8[1] = (char)(0x80 | ((c >> 6) & 0x3F)); - utf8[2] = (char)(0x80 | ( c & 0x3F)); - len = 3; - } - else { - utf8[0] = (char)(0xF0 | (c >> 18) ); - utf8[1] = (char)(0x80 | ((c >> 12) & 0x3F)); - utf8[2] = (char)(0x80 | ((c >> 6) & 0x3F)); - utf8[3] = (char)(0x80 | ( c & 0x3F)); - len = 4; - } + /* Unicode character (negative c indicates codepoint) */ + len = (int)mrb_utf8_to_buf(utf8, (uint32_t)(-c)); } if (p->tidx+len >= p->tsiz) { if (p->tsiz >= MRB_PARSER_TOKBUF_MAX) { diff --git a/mrbgems/mruby-compiler/core/y.tab.c b/mrbgems/mruby-compiler/core/y.tab.c index 4633c330b..e27edf8ee 100644 --- a/mrbgems/mruby-compiler/core/y.tab.c +++ b/mrbgems/mruby-compiler/core/y.tab.c @@ -12613,30 +12613,8 @@ tokadd(parser_state *p, int32_t c) len = 1; } else { - /* Unicode character */ - c = -c; - if (c < 0x80) { - utf8[0] = (char)c; - len = 1; - } - else if (c < 0x800) { - utf8[0] = (char)(0xC0 | (c >> 6)); - utf8[1] = (char)(0x80 | (c & 0x3F)); - len = 2; - } - else if (c < 0x10000) { - utf8[0] = (char)(0xE0 | (c >> 12) ); - utf8[1] = (char)(0x80 | ((c >> 6) & 0x3F)); - utf8[2] = (char)(0x80 | ( c & 0x3F)); - len = 3; - } - else { - utf8[0] = (char)(0xF0 | (c >> 18) ); - utf8[1] = (char)(0x80 | ((c >> 12) & 0x3F)); - utf8[2] = (char)(0x80 | ((c >> 6) & 0x3F)); - utf8[3] = (char)(0x80 | ( c & 0x3F)); - len = 4; - } + /* Unicode character (negative c indicates codepoint) */ + len = (int)mrb_utf8_to_buf(utf8, (uint32_t)(-c)); } if (p->tidx+len >= p->tsiz) { if (p->tsiz >= MRB_PARSER_TOKBUF_MAX) { diff --git a/mrbgems/mruby-io/src/io.c b/mrbgems/mruby-io/src/io.c index 46b35822a..ee9c8673e 100644 --- a/mrbgems/mruby-io/src/io.c +++ b/mrbgems/mruby-io/src/io.c @@ -1124,24 +1124,7 @@ io_putc(mrb_state *mrb, mrb_value io) if (len == 0) return c; #ifdef MRB_UTF8_STRING - /* Determine UTF-8 character length from first byte */ - unsigned char first = (unsigned char)ptr[0]; - if (first < 0x80) { - write_len = 1; /* ASCII */ - } - else if ((first & 0xE0) == 0xC0) { - write_len = 2; /* 2-byte UTF-8 */ - } - else if ((first & 0xF0) == 0xE0) { - write_len = 3; /* 3-byte UTF-8 */ - } - else if ((first & 0xF8) == 0xF0) { - write_len = 4; /* 4-byte UTF-8 */ - } - else { - write_len = 1; /* Invalid UTF-8, write single byte */ - } - if (write_len > len) write_len = len; + write_len = mrb_utf8len(ptr, ptr + len); #else write_len = 1; /* Non-UTF8: write single byte */ #endif diff --git a/mrbgems/mruby-pack/src/pack.c b/mrbgems/mruby-pack/src/pack.c index 4342c87d4..7688927a5 100644 --- a/mrbgems/mruby-pack/src/pack.c +++ b/mrbgems/mruby-pack/src/pack.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -772,36 +773,11 @@ static int pack_utf8(mrb_state *mrb, mrb_value o, mrb_value str, mrb_int sidx, int count, unsigned int flags) { char utf8[4]; - int len = 0; - uint32_t c = 0; + int len; + uint32_t c = (uint32_t)mrb_integer(o); - c = (uint32_t)mrb_integer(o); - - /* Unicode character */ - /* from mruby-compiler gem */ - if (c < 0x80) { - utf8[0] = (char)c; - len = 1; - } - else if (c < 0x800) { - utf8[0] = (char)(0xC0 | (c >> 6)); - utf8[1] = (char)(0x80 | (c & 0x3F)); - len = 2; - } - else if (c < 0x10000) { - utf8[0] = (char)(0xE0 | (c >> 12) ); - utf8[1] = (char)(0x80 | ((c >> 6) & 0x3F)); - utf8[2] = (char)(0x80 | ( c & 0x3F)); - len = 3; - } - else if (c < 0x200000) { - utf8[0] = (char)(0xF0 | (c >> 18) ); - utf8[1] = (char)(0x80 | ((c >> 12) & 0x3F)); - utf8[2] = (char)(0x80 | ((c >> 6) & 0x3F)); - utf8[3] = (char)(0x80 | ( c & 0x3F)); - len = 4; - } - else { + len = (int)mrb_utf8_to_buf(utf8, c); + if (len == 0) { mrb_raise(mrb, E_RANGE_ERROR, "pack(U): value out of range"); } diff --git a/mrbgems/mruby-sprintf/src/sprintf.c b/mrbgems/mruby-sprintf/src/sprintf.c index 5413f16ad..14dba5606 100644 --- a/mrbgems/mruby-sprintf/src/sprintf.c +++ b/mrbgems/mruby-sprintf/src/sprintf.c @@ -520,28 +520,8 @@ retry: /* Integer: encode directly to stack buffer (no allocation) */ mrb_int code = mrb_integer(val); #ifdef MRB_UTF8_STRING - if (code < 0x80) { - cbuf[0] = (char)code; - clen = 1; - } - else if (code < 0x800) { - cbuf[0] = (char)(0xC0 | (code >> 6)); - cbuf[1] = (char)(0x80 | (code & 0x3F)); - clen = 2; - } - else if (code < 0x10000) { - cbuf[0] = (char)(0xE0 | (code >> 12)); - cbuf[1] = (char)(0x80 | ((code >> 6) & 0x3F)); - cbuf[2] = (char)(0x80 | (code & 0x3F)); - clen = 3; - } - else { - cbuf[0] = (char)(0xF0 | (code >> 18)); - cbuf[1] = (char)(0x80 | ((code >> 12) & 0x3F)); - cbuf[2] = (char)(0x80 | ((code >> 6) & 0x3F)); - cbuf[3] = (char)(0x80 | (code & 0x3F)); - clen = 4; - } + clen = (int)mrb_utf8_to_buf(cbuf, (uint32_t)code); + if (clen == 0) clen = 1; /* invalid codepoint: write single byte */ #else cbuf[0] = (char)(code & 0xff); clen = 1; diff --git a/mrbgems/mruby-string-ext/src/string.c b/mrbgems/mruby-string-ext/src/string.c index 4e9c95d21..74dc1c7d1 100644 --- a/mrbgems/mruby-string-ext/src/string.c +++ b/mrbgems/mruby-string-ext/src/string.c @@ -51,36 +51,15 @@ int_chr_utf8(mrb_state *mrb, mrb_value num) char utf8[4]; mrb_int len; mrb_value str; - uint32_t sb_flag = 0; if (cp < 0 || 0x10FFFF < cp) { mrb_raisef(mrb, E_RANGE_ERROR, "%v out of char range", num); } - if (cp < 0x80) { - utf8[0] = (char)cp; - len = 1; - sb_flag = MRB_STR_SINGLE_BYTE; - } - else if (cp < 0x800) { - utf8[0] = (char)(0xC0 | (cp >> 6)); - utf8[1] = (char)(0x80 | (cp & 0x3F)); - len = 2; - } - else if (cp < 0x10000) { - utf8[0] = (char)(0xE0 | (cp >> 12)); - utf8[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); - utf8[2] = (char)(0x80 | ( cp & 0x3F)); - len = 3; - } - else { - utf8[0] = (char)(0xF0 | (cp >> 18)); - utf8[1] = (char)(0x80 | ((cp >> 12) & 0x3F)); - utf8[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); - utf8[3] = (char)(0x80 | ( cp & 0x3F)); - len = 4; - } + len = mrb_utf8_to_buf(utf8, (uint32_t)cp); str = mrb_str_new(mrb, utf8, len); - mrb_str_ptr(str)->flags |= sb_flag; + if (len == 1) { + RSTR_SET_ASCII_FLAG(mrb_str_ptr(str)); + } return str; } #endif diff --git a/src/string.c b/src/string.c index 768551d2d..cf758e66a 100644 --- a/src/string.c +++ b/src/string.c @@ -316,6 +316,39 @@ mrb_gc_free_str(mrb_state *mrb, struct RString *str) #define MASK01 0x01010101ul #endif +/* + * Encode a Unicode codepoint to UTF-8 bytes. + * buf must have at least 4 bytes of space. + * Returns the number of bytes written (1-4), or 0 for invalid codepoint. + */ +mrb_int +mrb_utf8_to_buf(char *buf, uint32_t cp) +{ + if (cp < 0x80) { + buf[0] = (char)cp; + return 1; + } + else if (cp < 0x800) { + buf[0] = (char)(0xC0 | (cp >> 6)); + buf[1] = (char)(0x80 | (cp & 0x3F)); + return 2; + } + else if (cp < 0x10000) { + buf[0] = (char)(0xE0 | (cp >> 12)); + buf[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); + buf[2] = (char)(0x80 | (cp & 0x3F)); + return 3; + } + else if (cp <= 0x10FFFF) { + buf[0] = (char)(0xF0 | (cp >> 18)); + buf[1] = (char)(0x80 | ((cp >> 12) & 0x3F)); + buf[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); + buf[3] = (char)(0x80 | (cp & 0x3F)); + return 4; + } + return 0; /* invalid codepoint */ +} + #ifdef MRB_UTF8_STRING #define NOASCII(c) ((c) & 0x80)