admin/mruby-mruby
1
0
Fork 0
mirror of https://github.com/mruby/mruby synced 2026-06-08 16:11:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

mruby-regexp: optimize Pike VM with pooled captures and generation counter

Browse Source 
Major changes to the NFA execution engine:

- Thread captures stored in a flat pool sized to actual ncap
  (e.g. 4 ints for 1 capture group vs 64 fixed), dramatically
  reducing per-thread copy cost
- Generation counter for visited[] eliminates per-step memset
  of the entire bytecode-length array
- Pool compaction between steps reclaims dead thread slots
- Backtracking engine also uses dynamic ncap-sized captures

Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Yukihiro "Matz" Matsumoto
2026-03-21 15:21:21 +09:00
parent 5a158d32a8
commit 4579caa7d8
1 changed files with 248 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mrbgems/mruby-regexp/src/re_exec.c
+248 -207
View File
@@ -10,146 +10,6 @@
#include "re_internal.h"
#include <string.h>
/* NFA thread: a position in the bytecode + captured positions */
typedef struct {
uint32_t pc;
int captures[RE_MAX_CAPTURES * 2];
} re_thread;
/* Thread list for NFA simulation */
typedef struct {
re_thread *threads;
int count;
int capa;
} re_threadlist;
/* Match result: updated when RE_MATCH is reached during epsilon traversal */
typedef struct {
mrb_bool matched;
int captures[RE_MAX_CAPTURES * 2];
} re_match_result;
static void
threadlist_init(mrb_state *mrb, re_threadlist *l, int capa)
{
l->threads = (re_thread*)mrb_malloc(mrb, sizeof(re_thread) * capa);
l->count = 0;
l->capa = capa;
}
static void
threadlist_free(mrb_state *mrb, re_threadlist *l)
{
mrb_free(mrb, l->threads);
}
/* Add a thread, following epsilon transitions (JMP, SPLIT, SAVE, assertions).
visited[] prevents adding duplicate threads at the same pc.
When RE_MATCH is reached, records in result and does NOT add to thread list. */
static void
add_thread(const mrb_regexp_pattern *pat, re_threadlist *list,
re_thread t, const char *str, const char *sp, const char *str_end,
uint8_t *visited, re_match_result *result)
{
for (;;) {
if (t.pc >= pat->code_len) return;
if (visited[t.pc]) return;
visited[t.pc] = 1;
re_inst inst = pat->code[t.pc];
switch (inst.op) {
case RE_JMP:
t.pc = inst.offset;
continue;
case RE_SPLIT:
/* greedy: try pc+1 first, then jump target */
{
re_thread t2 = t;
t2.pc = inst.offset;
add_thread(pat, list, t2, str, sp, str_end, visited, result);
}
t.pc++;
continue;
case RE_SPLITNG:
/* non-greedy: try jump target first, then pc+1 */
{
re_thread t2 = t;
t2.pc = t.pc + 1;
add_thread(pat, list, t2, str, sp, str_end, visited, result);
}
t.pc = inst.offset;
continue;
case RE_SAVE:
t.captures[inst.offset] = (int)(sp - str);
t.pc++;
continue;
case RE_BOL:
if (sp == str || ((pat->flags & RE_FLAG_MULTILINE) && sp > str && sp[-1] == '\n')) {
t.pc++; continue;
}
return;
case RE_EOL:
if (sp == str_end || ((pat->flags & RE_FLAG_MULTILINE) && *sp == '\n')) {
t.pc++; continue;
}
return;
case RE_BOT:
if (sp == str) { t.pc++; continue; }
return;
case RE_EOT:
if (sp == str_end) { t.pc++; continue; }
return;
case RE_EOTNL:
if (sp == str_end || (sp + 1 == str_end && *sp == '\n')) { t.pc++; continue; }
return;
case RE_WBOUND:
{
mrb_bool before = (sp > str) && re_is_word_char((uint8_t)sp[-1]);
mrb_bool after = (sp < str_end) && re_is_word_char((uint8_t)*sp);
if (before != after) { t.pc++; continue; }
}
return;
case RE_NWBOUND:
{
mrb_bool before = (sp > str) && re_is_word_char((uint8_t)sp[-1]);
mrb_bool after = (sp < str_end) && re_is_word_char((uint8_t)*sp);
if (before == after) { t.pc++; continue; }
}
return;
case RE_MATCH:
/* match found during epsilon traversal.
update result: later matches at same start position are longer
(greedy). thread ordering in add_thread ensures correct priority. */
if (result) {
result->matched = TRUE;
memcpy(result->captures, t.captures, sizeof(t.captures));
}
return; /* don't add to thread list */
default:
/* consuming instruction: add to thread list */
break;
}
break;
}
/* add to thread list */
if (list->count < list->capa) {
list->threads[list->count++] = t;
}
}
/* Check if character matches a character class */
static mrb_bool
class_match(const re_charclass *cc, uint8_t ch)
@@ -158,7 +18,162 @@ class_match(const re_charclass *cc, uint8_t ch)
return (cc->bitmap[ch >> 3] >> (ch & 7)) & 1;
}
/* Pike VM: NFA simulation with submatch tracking */
/*
* Pike VM with optimized thread storage.
*
* Key optimizations vs naive approach:
* - Captures stored in a flat pool, sized to actual ncap (not RE_MAX_CAPTURES)
* - Generation counter for visited[] eliminates per-step memset
* - Threads reference captures by pool index, avoiding 260-byte struct copies
*/
typedef struct {
uint32_t pc;
int cap_slot; /* slot index into capture pool */
} re_thread;
typedef struct {
re_thread *threads;
int count;
int capa;
} re_threadlist;
/* All Pike VM state */
typedef struct {
mrb_state *mrb;
const mrb_regexp_pattern *pat;
int ncap; /* actual capture count (num_captures * 2) */
int *cap_pool; /* flat: cap_pool[slot * ncap .. (slot+1) * ncap) */
int pool_next; /* next free slot */
int pool_capa; /* total slots allocated */
uint32_t *visited; /* generation-based */
uint32_t gen;
const char *str;
const char *str_end;
mrb_bool matched;
int *result_caps; /* best match (ncap ints) */
} pike_state;
static int
pool_alloc(pike_state *s)
{
if (s->pool_next >= s->pool_capa) {
int new_capa = s->pool_capa * 2;
s->cap_pool = (int*)mrb_realloc(s->mrb, s->cap_pool,
sizeof(int) * new_capa * s->ncap);
s->pool_capa = new_capa;
}
return s->pool_next++;
}
static int
pool_copy(pike_state *s, int src_slot)
{
int dst = pool_alloc(s);
memcpy(&s->cap_pool[dst * s->ncap],
&s->cap_pool[src_slot * s->ncap],
sizeof(int) * s->ncap);
return dst;
}
#define CAP(s, slot) (&(s)->cap_pool[(slot) * (s)->ncap])
/* Add thread following epsilon transitions.
visited[pc] == gen means already visited this step. */
static void
add_thread(pike_state *s, re_threadlist *list,
uint32_t pc, int cap_slot, const char *sp)
{
for (;;) {
if (pc >= s->pat->code_len) return;
if (s->visited[pc] == s->gen) return;
s->visited[pc] = s->gen;
re_inst inst = s->pat->code[pc];
switch (inst.op) {
case RE_JMP:
pc = inst.offset;
continue;
case RE_SPLIT:
{
int cp = pool_copy(s, cap_slot);
add_thread(s, list, inst.offset, cp, sp);
}
pc++;
continue;
case RE_SPLITNG:
{
int cp = pool_copy(s, cap_slot);
add_thread(s, list, pc + 1, cp, sp);
}
pc = inst.offset;
continue;
case RE_SAVE:
CAP(s, cap_slot)[inst.offset] = (int)(sp - s->str);
pc++;
continue;
case RE_BOL:
if (sp == s->str || ((s->pat->flags & RE_FLAG_MULTILINE) && sp > s->str && sp[-1] == '\n')) {
pc++; continue;
}
return;
case RE_EOL:
if (sp == s->str_end || ((s->pat->flags & RE_FLAG_MULTILINE) && *sp == '\n')) {
pc++; continue;
}
return;
case RE_BOT:
if (sp == s->str) { pc++; continue; }
return;
case RE_EOT:
if (sp == s->str_end) { pc++; continue; }
return;
case RE_EOTNL:
if (sp == s->str_end || (sp + 1 == s->str_end && *sp == '\n')) { pc++; continue; }
return;
case RE_WBOUND:
{
mrb_bool before = (sp > s->str) && re_is_word_char((uint8_t)sp[-1]);
mrb_bool after = (sp < s->str_end) && re_is_word_char((uint8_t)*sp);
if (before != after) { pc++; continue; }
}
return;
case RE_NWBOUND:
{
mrb_bool before = (sp > s->str) && re_is_word_char((uint8_t)sp[-1]);
mrb_bool after = (sp < s->str_end) && re_is_word_char((uint8_t)*sp);
if (before == after) { pc++; continue; }
}
return;
case RE_MATCH:
s->matched = TRUE;
memcpy(s->result_caps, CAP(s, cap_slot), sizeof(int) * s->ncap);
return;
default:
break;
}
break;
}
if (list->count < list->capa) {
re_thread *t = &list->threads[list->count++];
t->pc = pc;
t->cap_slot = cap_slot;
}
}
static int
pike_vm(mrb_state *mrb, const mrb_regexp_pattern *pat,
const char *str, mrb_int len, mrb_int start,
@@ -167,34 +182,54 @@ pike_vm(mrb_state *mrb, const mrb_regexp_pattern *pat,
const char *sp = str + start;
const char *str_end = str + len;
int ncap = pat->num_captures * 2;
if (ncap == 0) ncap = 2;
int list_capa = (int)pat->code_len * 2 + 16;
pike_state s;
s.mrb = mrb;
s.pat = pat;
s.ncap = ncap;
s.str = str;
s.str_end = str_end;
s.matched = FALSE;
s.gen = 1;
s.pool_capa = list_capa * 2;
s.pool_next = 0;
s.cap_pool = (int*)mrb_malloc(mrb, sizeof(int) * s.pool_capa * ncap);
s.result_caps = (int*)mrb_malloc(mrb, sizeof(int) * ncap);
memset(s.result_caps, -1, sizeof(int) * ncap);
s.visited = (uint32_t*)mrb_calloc(mrb, pat->code_len + 1, sizeof(uint32_t));
re_threadlist curr, next;
threadlist_init(mrb, &curr, list_capa);
threadlist_init(mrb, &next, list_capa);
uint8_t *visited = (uint8_t*)mrb_calloc(mrb, 1, pat->code_len + 1);
re_match_result result;
result.matched = FALSE;
memset(result.captures, -1, sizeof(result.captures));
curr.threads = (re_thread*)mrb_malloc(mrb, sizeof(re_thread) * list_capa);
curr.count = 0; curr.capa = list_capa;
next.threads = (re_thread*)mrb_malloc(mrb, sizeof(re_thread) * list_capa);
next.count = 0; next.capa = list_capa;
for (; sp <= str_end; sp++) {
/* Add a new initial thread at current position (unanchored search) */
if (!result.matched) {
re_thread t0;
memset(t0.captures, -1, sizeof(t0.captures));
t0.pc = 0;
memset(visited, 0, pat->code_len + 1);
add_thread(pat, &curr, t0, str, sp, str_end, visited, &result);
/* if match found during epsilon traversal (empty pattern), done */
if (result.matched && curr.count == 0) break;
if (!s.matched) {
int slot = pool_alloc(&s);
memset(CAP(&s, slot), -1, sizeof(int) * ncap);
s.gen++;
add_thread(&s, &curr, 0, slot, sp);
if (s.matched && curr.count == 0) break;
}
if (sp >= str_end) break;
/* Process all current threads against current character */
memset(visited, 0, pat->code_len + 1);
/* Reset pool for next step.
First, compact: copy live thread captures to the front of the pool. */
for (int i = 0; i < curr.count; i++) {
if (curr.threads[i].cap_slot != i) {
memcpy(CAP(&s, i), CAP(&s, curr.threads[i].cap_slot),
sizeof(int) * ncap);
curr.threads[i].cap_slot = i;
}
}
s.pool_next = curr.count;
s.gen++;
next.count = 0;
int ch = (uint8_t)*sp;
@@ -208,41 +243,39 @@ pike_vm(mrb_state *mrb, const mrb_regexp_pattern *pat,
switch (inst.op) {
case RE_CHAR:
if (ch == inst.a) {
th->pc++;
add_thread(pat, &next, *th, str, sp + 1, str_end, visited, &result);
int cp = pool_copy(&s, th->cap_slot);
add_thread(&s, &next, th->pc + 1, cp, sp + 1);
}
break;
case RE_ANY:
if (ch != '\n') {
th->pc++;
add_thread(pat, &next, *th, str, sp + advance, str_end, visited, &result);
int cp = pool_copy(&s, th->cap_slot);
add_thread(&s, &next, th->pc + 1, cp, sp + advance);
}
break;
case RE_ANY_NL:
th->pc++;
add_thread(pat, &next, *th, str, sp + advance, str_end, visited, &result);
{
int cp = pool_copy(&s, th->cap_slot);
add_thread(&s, &next, th->pc + 1, cp, sp + advance);
}
break;
case RE_CLASS:
if (class_match(&pat->classes[inst.a], (uint8_t)ch)) {
th->pc++;
add_thread(pat, &next, *th, str, sp + advance, str_end, visited, &result);
int cp = pool_copy(&s, th->cap_slot);
add_thread(&s, &next, th->pc + 1, cp, sp + advance);
}
break;
case RE_NCLASS:
if (!class_match(&pat->classes[inst.a], (uint8_t)ch)) {
th->pc++;
add_thread(pat, &next, *th, str, sp + advance, str_end, visited, &result);
int cp = pool_copy(&s, th->cap_slot);
add_thread(&s, &next, th->pc + 1, cp, sp + advance);
}
break;
case RE_BACKREF:
/* TODO: backtracking for backreferences */
break;
default:
break;
}
@@ -255,29 +288,34 @@ pike_vm(mrb_state *mrb, const mrb_regexp_pattern *pat,
next = tmp;
}
/* if matched and no more threads, we're done */
if (result.matched && curr.count == 0) break;
if (s.matched && curr.count == 0) break;
}
threadlist_free(mrb, &curr);
threadlist_free(mrb, &next);
mrb_free(mrb, visited);
if (result.matched && captures) {
int copy = ncap < captures_size ? ncap : captures_size;
memcpy(captures, result.captures, sizeof(int) * copy);
int ret = 0;
if (s.matched) {
if (captures) {
int copy = ncap < captures_size ? ncap : captures_size;
memcpy(captures, s.result_caps, sizeof(int) * copy);
}
ret = ncap > 0 ? ncap : 1;
}
return result.matched ? (ncap > 0 ? ncap : 1) : 0;
mrb_free(mrb, curr.threads);
mrb_free(mrb, next.threads);
mrb_free(mrb, s.cap_pool);
mrb_free(mrb, s.result_caps);
mrb_free(mrb, s.visited);
return ret;
}
/*
* Backtracking engine for patterns with backreferences.
* Recursive: tries each possibility and backtracks on failure.
* Step-limited to prevent ReDoS.
*/
static mrb_bool
bt_match(const mrb_regexp_pattern *pat, const char *str, const char *str_end,
const char *sp, uint32_t pc, int *captures, int *steps)
const char *sp, uint32_t pc, int *captures, int ncap, int *steps)
{
while (pc < pat->code_len) {
if (++(*steps) > MRB_REGEXP_STEP_LIMIT) return FALSE;
@@ -317,23 +355,24 @@ bt_match(const mrb_regexp_pattern *pat, const char *str, const char *str_end,
break;
case RE_SPLIT:
/* greedy: try pc+1 first */
if (bt_match(pat, str, str_end, sp, pc + 1, captures, steps)) return TRUE;
if (bt_match(pat, str, str_end, sp, pc + 1, captures, ncap, steps)) return TRUE;
pc = inst.offset;
break;
case RE_SPLITNG:
/* non-greedy: try offset first */
if (bt_match(pat, str, str_end, sp, inst.offset, captures, steps)) return TRUE;
if (bt_match(pat, str, str_end, sp, inst.offset, captures, ncap, steps)) return TRUE;
pc++;
break;
case RE_SAVE:
{
int old = captures[inst.offset];
captures[inst.offset] = (int)(sp - str);
if (bt_match(pat, str, str_end, sp, pc + 1, captures, steps)) return TRUE;
captures[inst.offset] = old; /* restore on backtrack */
int slot = inst.offset;
if (slot < ncap) {
int old = captures[slot];
captures[slot] = (int)(sp - str);
if (bt_match(pat, str, str_end, sp, pc + 1, captures, ncap, steps)) return TRUE;
captures[slot] = old;
}
return FALSE;
}
@@ -381,24 +420,22 @@ bt_match(const mrb_regexp_pattern *pat, const char *str, const char *str_end,
int gs = captures[group * 2];
int ge = captures[group * 2 + 1];
if (gs < 0 || ge < 0) return FALSE;
int len = ge - gs;
if (sp + len > str_end) return FALSE;
if (memcmp(sp, str + gs, len) != 0) return FALSE;
sp += len;
int blen = ge - gs;
if (sp + blen > str_end) return FALSE;
if (memcmp(sp, str + gs, blen) != 0) return FALSE;
sp += blen;
pc++;
}
break;
case RE_LOOKAHEAD:
/* positive lookahead: sub-pattern must match at current position */
if (!bt_match(pat, str, str_end, sp, pc + 1, captures, steps))
if (!bt_match(pat, str, str_end, sp, pc + 1, captures, ncap, steps))
return FALSE;
pc = inst.offset; /* skip past sub-pattern */
pc = inst.offset;
break;
case RE_NEG_LOOKAHEAD:
/* negative lookahead: sub-pattern must NOT match */
if (bt_match(pat, str, str_end, sp, pc + 1, captures, steps))
if (bt_match(pat, str, str_end, sp, pc + 1, captures, ncap, steps))
return FALSE;
pc = inst.offset;
break;
@@ -417,24 +454,28 @@ backtrack_exec(mrb_state *mrb, const mrb_regexp_pattern *pat,
{
const char *str_end = str + len;
int ncap = pat->num_captures * 2;
if (ncap == 0) ncap = 2;
int *caps = (int*)mrb_malloc(mrb, sizeof(int) * ncap);
for (const char *sp = str + start; sp <= str_end; sp++) {
int caps[RE_MAX_CAPTURES * 2];
memset(caps, -1, sizeof(caps));
memset(caps, -1, sizeof(int) * ncap);
int steps = 0;
if (bt_match(pat, str, str_end, sp, 0, caps, &steps)) {
if (bt_match(pat, str, str_end, sp, 0, caps, ncap, &steps)) {
if (captures) {
int copy = ncap < captures_size ? ncap : captures_size;
memcpy(captures, caps, sizeof(int) * copy);
}
mrb_free(mrb, caps);
return ncap > 0 ? ncap : 1;
}
}
mrb_free(mrb, caps);
return 0;
}
/* Public entry point: dispatch to Pike VM or backtracking engine */
/* Public entry point */
int
re_exec(mrb_state *mrb, const mrb_regexp_pattern *pat,
const char *str, mrb_int len, mrb_int start,