From 78d761addf7d2adddc1c9cb69ef6e3964ff098b5 Mon Sep 17 00:00:00 2001 From: "Yukihiro \"Matz\" Matsumoto" Date: Sat, 21 Mar 2026 14:19:35 +0900 Subject: [PATCH] mruby-regexp: implement extended mode (x flag) The x flag ignores unescaped whitespace and #comments in patterns, making complex regexps more readable. Whitespace inside character classes [...] remains literal. Implemented as a preprocessing step that strips whitespace/comments before compilation. Co-authored-by: Claude --- mrbgems/mruby-regexp/include/re_internal.h | 3 +- mrbgems/mruby-regexp/src/re_compile.c | 56 ++++++++++++++++++++++ mrbgems/mruby-regexp/src/regexp.c | 5 +- mrbgems/mruby-regexp/test/regexp.rb | 25 ++++++++++ 4 files changed, 87 insertions(+), 2 deletions(-) diff --git a/mrbgems/mruby-regexp/include/re_internal.h b/mrbgems/mruby-regexp/include/re_internal.h index 6fbab3c0e..6cda84386 100644 --- a/mrbgems/mruby-regexp/include/re_internal.h +++ b/mrbgems/mruby-regexp/include/re_internal.h @@ -74,9 +74,10 @@ typedef struct mrb_regexp_pattern { #define RE_FLAG_IGNORECASE 1 #define RE_FLAG_MULTILINE 2 /* ^ and $ match at \n boundaries */ #define RE_FLAG_DOTALL 4 /* . matches \n (Ruby's /m for dot behavior) */ +#define RE_FLAG_EXTENDED 8 /* ignore whitespace and #comments in pattern */ /* Note: Ruby's /m flag means BOTH multiline anchors AND dotall. - Ruby's /i flag is ignorecase. */ + Ruby's /i flag is ignorecase. Ruby's /x flag is extended. */ /* Step limit for ReDoS protection */ #ifndef MRB_REGEXP_STEP_LIMIT diff --git a/mrbgems/mruby-regexp/src/re_compile.c b/mrbgems/mruby-regexp/src/re_compile.c index 504f9020d..403b3a171 100644 --- a/mrbgems/mruby-regexp/src/re_compile.c +++ b/mrbgems/mruby-regexp/src/re_compile.c @@ -29,6 +29,7 @@ typedef struct { uint16_t num_named; mrb_bool has_backref; mrb_bool has_nongreedy; + char *stripped; /* allocated buffer for x-mode preprocessing */ } re_compiler; static void compile_alt(re_compiler *c); /* forward */ @@ -36,6 +37,8 @@ static void compile_alt(re_compiler *c); /* forward */ static void compile_error(re_compiler *c, const char *msg) { + if (c->stripped) mrb_free(c->mrb, c->stripped); + c->stripped = NULL; mrb_raisef(c->mrb, mrb_exc_get_id(c->mrb, MRB_SYM(RegexpError)), "%s: /%s/", msg, c->src); } @@ -606,11 +609,63 @@ compile_alt(re_compiler *c) } } +/* + * Strip whitespace and #comments for extended mode (/x flag). + * Whitespace inside [...] character classes is preserved. + * Escaped characters (\ followed by anything) are preserved. + */ +static char* +strip_extended(mrb_state *mrb, const char *src, mrb_int len, mrb_int *out_len) +{ + char *buf = (char*)mrb_malloc(mrb, len); + mrb_int o = 0; + mrb_bool in_class = FALSE; + const char *end = src + len; + + while (src < end) { + char ch = *src; + if (ch == '\\' && src + 1 < end) { + buf[o++] = *src++; + buf[o++] = *src++; + continue; + } + if (in_class) { + if (ch == ']') in_class = FALSE; + buf[o++] = *src++; + continue; + } + if (ch == '[') { + in_class = TRUE; + buf[o++] = *src++; + continue; + } + if (ch == '#') { + /* skip to end of line */ + while (src < end && *src != '\n') src++; + continue; + } + if (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' || ch == '\f' || ch == '\v') { + src++; + continue; + } + buf[o++] = *src++; + } + *out_len = o; + return buf; +} + mrb_regexp_pattern* re_compile(mrb_state *mrb, const char *pattern, mrb_int len, uint32_t flags) { re_compiler c; memset(&c, 0, sizeof(c)); + + if (flags & RE_FLAG_EXTENDED) { + mrb_int slen; + c.stripped = strip_extended(mrb, pattern, len, &slen); + pattern = c.stripped; + len = slen; + } c.mrb = mrb; c.src = pattern; c.src_end = pattern + len; @@ -643,6 +698,7 @@ re_compile(mrb_state *mrb, const char *pattern, mrb_int len, uint32_t flags) pat->has_backref = c.has_backref; pat->has_nongreedy = c.has_nongreedy; + if (c.stripped) mrb_free(mrb, c.stripped); return pat; } diff --git a/mrbgems/mruby-regexp/src/regexp.c b/mrbgems/mruby-regexp/src/regexp.c index e757a6b3a..77c0fe4dd 100644 --- a/mrbgems/mruby-regexp/src/regexp.c +++ b/mrbgems/mruby-regexp/src/regexp.c @@ -50,6 +50,7 @@ parse_flags(mrb_state *mrb, mrb_value flags_val) if (mrb_integer_p(flags_val)) { mrb_int f = mrb_integer(flags_val); if (f & 1) flags |= RE_FLAG_IGNORECASE; + if (f & 2) flags |= RE_FLAG_EXTENDED; if (f & 4) flags |= RE_FLAG_MULTILINE | RE_FLAG_DOTALL; return flags; } @@ -60,7 +61,7 @@ parse_flags(mrb_state *mrb, mrb_value flags_val) switch (s[i]) { case 'i': flags |= RE_FLAG_IGNORECASE; break; case 'm': flags |= RE_FLAG_MULTILINE | RE_FLAG_DOTALL; break; - case 'x': break; /* TODO: extended mode */ + case 'x': flags |= RE_FLAG_EXTENDED; break; } } return flags; @@ -265,6 +266,7 @@ regexp_to_s(mrb_state *mrb, mrb_value self) mrb_value result = mrb_str_new_lit(mrb, "(?"); if (flags & RE_FLAG_IGNORECASE) mrb_str_cat_lit(mrb, result, "i"); if (flags & RE_FLAG_MULTILINE) mrb_str_cat_lit(mrb, result, "m"); + if (flags & RE_FLAG_EXTENDED) mrb_str_cat_lit(mrb, result, "x"); mrb_str_cat_lit(mrb, result, ":"); mrb_str_cat_str(mrb, result, src); mrb_str_cat_lit(mrb, result, ")"); @@ -283,6 +285,7 @@ regexp_inspect(mrb_state *mrb, mrb_value self) mrb_str_cat_lit(mrb, result, "/"); if (flags & RE_FLAG_IGNORECASE) mrb_str_cat_lit(mrb, result, "i"); if (flags & RE_FLAG_MULTILINE) mrb_str_cat_lit(mrb, result, "m"); + if (flags & RE_FLAG_EXTENDED) mrb_str_cat_lit(mrb, result, "x"); return result; } diff --git a/mrbgems/mruby-regexp/test/regexp.rb b/mrbgems/mruby-regexp/test/regexp.rb index 7520cec76..58abf2ff4 100644 --- a/mrbgems/mruby-regexp/test/regexp.rb +++ b/mrbgems/mruby-regexp/test/regexp.rb @@ -149,6 +149,31 @@ assert("Regexp#hash") do assert_not_equal r1.hash, r3.hash end +assert("Regexp extended mode (x flag)") do + # whitespace is ignored + re = Regexp.new('a b c', Regexp::EXTENDED) + assert_true re.match?("abc") + assert_false re.match?("a b c") + + # comments are ignored + re = Regexp.new("a # match a\nb # match b\nc", Regexp::EXTENDED) + assert_true re.match?("abc") + + # whitespace inside character class is literal + re = Regexp.new('[ ]', Regexp::EXTENDED) + assert_true re.match?(" ") + + # escaped whitespace is preserved + re = Regexp.new('a\\ b', Regexp::EXTENDED) + assert_true re.match?("a b") + + # inspect shows x flag + assert_equal "/abc/x", Regexp.new("abc", Regexp::EXTENDED).inspect + + # to_s shows x flag + assert_equal "(?x:abc)", Regexp.new("abc", Regexp::EXTENDED).to_s +end + assert("String#match") do md = "hello world".match(Regexp.new("(\\w+)\\s(\\w+)")) assert_equal "hello", md[1]