From 337b5906a238bb5325400fe73a3fb77924fcba89 Mon Sep 17 00:00:00 2001 From: "Yukihiro \"Matz\" Matsumoto" Date: Mon, 23 Mar 2026 10:25:15 +0900 Subject: [PATCH] mruby-regexp: add first-byte bitmap for fast position skipping Compute a 128-bit bitmap of bytes that could start a match. For patterns like /cat|dog|fox/ the bitmap contains only {b,c,d,f}, skipping positions where no alternative can match. For /\d+/ only {'0'-'9'} are set. Used when no literal prefix is available (alternation, character class patterns). Falls back gracefully when too many bytes match. Key improvements vs CRuby ratio: alternation miss: 9.5x -> 3.6x \d+ medium string: 3.0x -> 1.4x Co-authored-by: Claude --- mrbgems/mruby-regexp/include/re_internal.h | 2 + mrbgems/mruby-regexp/src/re_compile.c | 87 ++++++++++++++++++++++ mrbgems/mruby-regexp/src/re_exec.c | 26 +++++-- 3 files changed, 109 insertions(+), 6 deletions(-) diff --git a/mrbgems/mruby-regexp/include/re_internal.h b/mrbgems/mruby-regexp/include/re_internal.h index 7f2bd9439..af81f108e 100644 --- a/mrbgems/mruby-regexp/include/re_internal.h +++ b/mrbgems/mruby-regexp/include/re_internal.h @@ -72,6 +72,8 @@ typedef struct mrb_regexp_pattern { mrb_bool needs_backtrack; /* true if pattern needs backtracking engine */ uint8_t *prefix; /* literal prefix bytes for fast skip (or NULL) */ uint8_t prefix_len; /* length of prefix (0 = no prefix) */ + uint8_t first_bytes[16]; /* bitmap of possible first bytes (128-bit, ASCII) */ + mrb_bool has_first_bytes; /* true if first_bytes is usable for skipping */ } mrb_regexp_pattern; /* Regexp flags */ diff --git a/mrbgems/mruby-regexp/src/re_compile.c b/mrbgems/mruby-regexp/src/re_compile.c index e003b2b6d..79d01aacb 100644 --- a/mrbgems/mruby-regexp/src/re_compile.c +++ b/mrbgems/mruby-regexp/src/re_compile.c @@ -732,6 +732,82 @@ strip_extended(mrb_state *mrb, const char *src, mrb_int len, mrb_int *out_len) return buf; } +/* + * Compute the set of bytes that could be the first consumed byte of a match. + * Walks bytecode from pc=0, following epsilon transitions (SAVE, JMP, SPLIT). + * Returns TRUE if the set is narrower than "any byte" (i.e., useful for skip). + */ +static mrb_bool +first_set_walk(const re_inst *code, uint32_t code_len, + const re_charclass *classes, uint32_t pc, + uint8_t *bm, uint8_t *seen) +{ + while (pc < code_len) { + if (seen[pc]) return TRUE; /* already visited */ + seen[pc] = 1; + switch (code[pc].op) { + case RE_SAVE: + case RE_BOL: case RE_EOL: case RE_BOT: case RE_EOT: case RE_EOTNL: + case RE_WBOUND: case RE_NWBOUND: + pc++; + continue; /* zero-width, keep walking */ + case RE_JMP: + pc = code[pc].offset; + continue; + case RE_SPLIT: + /* both branches: pc+1 and offset */ + if (!first_set_walk(code, code_len, classes, code[pc].offset, bm, seen)) + return FALSE; + pc++; + continue; + case RE_SPLITNG: + if (!first_set_walk(code, code_len, classes, pc + 1, bm, seen)) + return FALSE; + pc = code[pc].offset; + continue; + case RE_CHAR: + bm[code[pc].a >> 3] |= (1 << (code[pc].a & 7)); + return TRUE; + case RE_CLASS: { + const re_charclass *cc = &classes[code[pc].a]; + for (int i = 0; i < 16; i++) bm[i] |= cc->bitmap[i]; + if (cc->utf8_any) return FALSE; /* non-ASCII possible */ + return TRUE; + } + case RE_NCLASS: { + /* negated class: complement of bitmap. Too many bits; not useful. */ + return FALSE; + } + case RE_ANY: case RE_ANY_NL: + return FALSE; /* any byte possible */ + case RE_MATCH: + return TRUE; /* empty match; first_bytes still valid for other branches */ + default: + return FALSE; + } + } + return TRUE; +} + +static mrb_bool +compute_first_set(const re_inst *code, uint32_t code_len, + const re_charclass *classes, uint8_t *bm) +{ + uint8_t seen[4096]; + if (code_len >= sizeof(seen)) return FALSE; /* pattern too large */ + memset(seen, 0, code_len + 1); + if (!first_set_walk(code, code_len, classes, 0, bm, seen)) + return FALSE; + /* Check if bitmap is all-ones (no benefit to skip) */ + int set_bits = 0; + for (int i = 0; i < 16; i++) { + for (int b = 0; b < 8; b++) { + if (bm[i] & (1 << b)) set_bits++; + } + } + return set_bits < 96; /* useful only if fewer than 75% of bytes match */ +} + mrb_regexp_pattern* re_compile(mrb_state *mrb, const char *pattern, mrb_int len, uint32_t flags) { @@ -799,6 +875,17 @@ re_compile(mrb_state *mrb, const char *pattern, mrb_int len, uint32_t flags) } } + /* Compute first-byte bitmap: set of bytes that could start a match. + Used when prefix is empty (e.g. alternation, character class patterns). */ + { + uint8_t bm[16]; + memset(bm, 0, sizeof(bm)); + pat->has_first_bytes = compute_first_set(pat->code, pat->code_len, pat->classes, bm); + if (pat->has_first_bytes) { + memcpy(pat->first_bytes, bm, 16); + } + } + if (c.stripped) mrb_free(mrb, c.stripped); return pat; } diff --git a/mrbgems/mruby-regexp/src/re_exec.c b/mrbgems/mruby-regexp/src/re_exec.c index 98252bc6f..662ad67a1 100644 --- a/mrbgems/mruby-regexp/src/re_exec.c +++ b/mrbgems/mruby-regexp/src/re_exec.c @@ -34,6 +34,10 @@ skip_to_prefix(const mrb_regexp_pattern *pat, const char *sp, const char *str_en return NULL; } +/* Check if a byte is in the first-byte bitmap */ +#define FIRST_BYTE_OK(pat, ch) \ + ((ch) >= 128 || ((pat)->first_bytes[(ch) >> 3] & (1 << ((ch) & 7)))) + /* Check if character matches a character class */ static mrb_bool class_match(const re_charclass *cc, uint8_t ch) @@ -250,11 +254,17 @@ pike_vm(mrb_state *mrb, const mrb_regexp_pattern *pat, for (; sp <= str_end; sp++) { if (!s.matched) { - /* Skip ahead using literal prefix when no active threads */ - if (pat->prefix_len > 0 && curr.count == 0) { - const char *skip = skip_to_prefix(pat, sp, str_end); - if (!skip) break; /* prefix not found anywhere; no match possible */ - sp = skip; + /* Skip ahead when no active threads */ + if (curr.count == 0) { + if (pat->prefix_len > 0) { + const char *skip = skip_to_prefix(pat, sp, str_end); + if (!skip) break; + sp = skip; + } + else if (pat->has_first_bytes) { + while (sp < str_end && !FIRST_BYTE_OK(pat, (uint8_t)*sp)) sp++; + if (sp > str_end) break; + } } int slot = match_only ? 0 : pool_alloc(&s); if (!match_only) memset(CAP(&s, slot), -1, sizeof(int) * ncap); @@ -529,12 +539,16 @@ backtrack_exec(mrb_state *mrb, const mrb_regexp_pattern *pat, int *caps = (int*)mrb_malloc(mrb, sizeof(int) * ncap); for (const char *sp = str + start; sp <= str_end; sp++) { - /* Skip ahead using literal prefix */ + /* Skip ahead using literal prefix or first-byte bitmap */ if (pat->prefix_len > 0) { const char *skip = skip_to_prefix(pat, sp, str_end); if (!skip) break; sp = skip; } + else if (pat->has_first_bytes) { + while (sp < str_end && !FIRST_BYTE_OK(pat, (uint8_t)*sp)) sp++; + if (sp > str_end) break; + } memset(caps, -1, sizeof(int) * ncap); int steps = 0;