diff --git a/mrbgems/mruby-regexp/src/re_exec.c b/mrbgems/mruby-regexp/src/re_exec.c index 18d2de5c2..ad3a6d7db 100644 --- a/mrbgems/mruby-regexp/src/re_exec.c +++ b/mrbgems/mruby-regexp/src/re_exec.c @@ -10,146 +10,6 @@ #include "re_internal.h" #include -/* NFA thread: a position in the bytecode + captured positions */ -typedef struct { - uint32_t pc; - int captures[RE_MAX_CAPTURES * 2]; -} re_thread; - -/* Thread list for NFA simulation */ -typedef struct { - re_thread *threads; - int count; - int capa; -} re_threadlist; - -/* Match result: updated when RE_MATCH is reached during epsilon traversal */ -typedef struct { - mrb_bool matched; - int captures[RE_MAX_CAPTURES * 2]; -} re_match_result; - -static void -threadlist_init(mrb_state *mrb, re_threadlist *l, int capa) -{ - l->threads = (re_thread*)mrb_malloc(mrb, sizeof(re_thread) * capa); - l->count = 0; - l->capa = capa; -} - -static void -threadlist_free(mrb_state *mrb, re_threadlist *l) -{ - mrb_free(mrb, l->threads); -} - -/* Add a thread, following epsilon transitions (JMP, SPLIT, SAVE, assertions). - visited[] prevents adding duplicate threads at the same pc. - When RE_MATCH is reached, records in result and does NOT add to thread list. */ -static void -add_thread(const mrb_regexp_pattern *pat, re_threadlist *list, - re_thread t, const char *str, const char *sp, const char *str_end, - uint8_t *visited, re_match_result *result) -{ - for (;;) { - if (t.pc >= pat->code_len) return; - if (visited[t.pc]) return; - visited[t.pc] = 1; - - re_inst inst = pat->code[t.pc]; - switch (inst.op) { - case RE_JMP: - t.pc = inst.offset; - continue; - - case RE_SPLIT: - /* greedy: try pc+1 first, then jump target */ - { - re_thread t2 = t; - t2.pc = inst.offset; - add_thread(pat, list, t2, str, sp, str_end, visited, result); - } - t.pc++; - continue; - - case RE_SPLITNG: - /* non-greedy: try jump target first, then pc+1 */ - { - re_thread t2 = t; - t2.pc = t.pc + 1; - add_thread(pat, list, t2, str, sp, str_end, visited, result); - } - t.pc = inst.offset; - continue; - - case RE_SAVE: - t.captures[inst.offset] = (int)(sp - str); - t.pc++; - continue; - - case RE_BOL: - if (sp == str || ((pat->flags & RE_FLAG_MULTILINE) && sp > str && sp[-1] == '\n')) { - t.pc++; continue; - } - return; - - case RE_EOL: - if (sp == str_end || ((pat->flags & RE_FLAG_MULTILINE) && *sp == '\n')) { - t.pc++; continue; - } - return; - - case RE_BOT: - if (sp == str) { t.pc++; continue; } - return; - - case RE_EOT: - if (sp == str_end) { t.pc++; continue; } - return; - - case RE_EOTNL: - if (sp == str_end || (sp + 1 == str_end && *sp == '\n')) { t.pc++; continue; } - return; - - case RE_WBOUND: - { - mrb_bool before = (sp > str) && re_is_word_char((uint8_t)sp[-1]); - mrb_bool after = (sp < str_end) && re_is_word_char((uint8_t)*sp); - if (before != after) { t.pc++; continue; } - } - return; - - case RE_NWBOUND: - { - mrb_bool before = (sp > str) && re_is_word_char((uint8_t)sp[-1]); - mrb_bool after = (sp < str_end) && re_is_word_char((uint8_t)*sp); - if (before == after) { t.pc++; continue; } - } - return; - - case RE_MATCH: - /* match found during epsilon traversal. - update result: later matches at same start position are longer - (greedy). thread ordering in add_thread ensures correct priority. */ - if (result) { - result->matched = TRUE; - memcpy(result->captures, t.captures, sizeof(t.captures)); - } - return; /* don't add to thread list */ - - default: - /* consuming instruction: add to thread list */ - break; - } - break; - } - - /* add to thread list */ - if (list->count < list->capa) { - list->threads[list->count++] = t; - } -} - /* Check if character matches a character class */ static mrb_bool class_match(const re_charclass *cc, uint8_t ch) @@ -158,7 +18,162 @@ class_match(const re_charclass *cc, uint8_t ch) return (cc->bitmap[ch >> 3] >> (ch & 7)) & 1; } -/* Pike VM: NFA simulation with submatch tracking */ +/* + * Pike VM with optimized thread storage. + * + * Key optimizations vs naive approach: + * - Captures stored in a flat pool, sized to actual ncap (not RE_MAX_CAPTURES) + * - Generation counter for visited[] eliminates per-step memset + * - Threads reference captures by pool index, avoiding 260-byte struct copies + */ + +typedef struct { + uint32_t pc; + int cap_slot; /* slot index into capture pool */ +} re_thread; + +typedef struct { + re_thread *threads; + int count; + int capa; +} re_threadlist; + +/* All Pike VM state */ +typedef struct { + mrb_state *mrb; + const mrb_regexp_pattern *pat; + int ncap; /* actual capture count (num_captures * 2) */ + int *cap_pool; /* flat: cap_pool[slot * ncap .. (slot+1) * ncap) */ + int pool_next; /* next free slot */ + int pool_capa; /* total slots allocated */ + uint32_t *visited; /* generation-based */ + uint32_t gen; + const char *str; + const char *str_end; + mrb_bool matched; + int *result_caps; /* best match (ncap ints) */ +} pike_state; + +static int +pool_alloc(pike_state *s) +{ + if (s->pool_next >= s->pool_capa) { + int new_capa = s->pool_capa * 2; + s->cap_pool = (int*)mrb_realloc(s->mrb, s->cap_pool, + sizeof(int) * new_capa * s->ncap); + s->pool_capa = new_capa; + } + return s->pool_next++; +} + +static int +pool_copy(pike_state *s, int src_slot) +{ + int dst = pool_alloc(s); + memcpy(&s->cap_pool[dst * s->ncap], + &s->cap_pool[src_slot * s->ncap], + sizeof(int) * s->ncap); + return dst; +} + +#define CAP(s, slot) (&(s)->cap_pool[(slot) * (s)->ncap]) + +/* Add thread following epsilon transitions. + visited[pc] == gen means already visited this step. */ +static void +add_thread(pike_state *s, re_threadlist *list, + uint32_t pc, int cap_slot, const char *sp) +{ + for (;;) { + if (pc >= s->pat->code_len) return; + if (s->visited[pc] == s->gen) return; + s->visited[pc] = s->gen; + + re_inst inst = s->pat->code[pc]; + switch (inst.op) { + case RE_JMP: + pc = inst.offset; + continue; + + case RE_SPLIT: + { + int cp = pool_copy(s, cap_slot); + add_thread(s, list, inst.offset, cp, sp); + } + pc++; + continue; + + case RE_SPLITNG: + { + int cp = pool_copy(s, cap_slot); + add_thread(s, list, pc + 1, cp, sp); + } + pc = inst.offset; + continue; + + case RE_SAVE: + CAP(s, cap_slot)[inst.offset] = (int)(sp - s->str); + pc++; + continue; + + case RE_BOL: + if (sp == s->str || ((s->pat->flags & RE_FLAG_MULTILINE) && sp > s->str && sp[-1] == '\n')) { + pc++; continue; + } + return; + + case RE_EOL: + if (sp == s->str_end || ((s->pat->flags & RE_FLAG_MULTILINE) && *sp == '\n')) { + pc++; continue; + } + return; + + case RE_BOT: + if (sp == s->str) { pc++; continue; } + return; + + case RE_EOT: + if (sp == s->str_end) { pc++; continue; } + return; + + case RE_EOTNL: + if (sp == s->str_end || (sp + 1 == s->str_end && *sp == '\n')) { pc++; continue; } + return; + + case RE_WBOUND: + { + mrb_bool before = (sp > s->str) && re_is_word_char((uint8_t)sp[-1]); + mrb_bool after = (sp < s->str_end) && re_is_word_char((uint8_t)*sp); + if (before != after) { pc++; continue; } + } + return; + + case RE_NWBOUND: + { + mrb_bool before = (sp > s->str) && re_is_word_char((uint8_t)sp[-1]); + mrb_bool after = (sp < s->str_end) && re_is_word_char((uint8_t)*sp); + if (before == after) { pc++; continue; } + } + return; + + case RE_MATCH: + s->matched = TRUE; + memcpy(s->result_caps, CAP(s, cap_slot), sizeof(int) * s->ncap); + return; + + default: + break; + } + break; + } + + if (list->count < list->capa) { + re_thread *t = &list->threads[list->count++]; + t->pc = pc; + t->cap_slot = cap_slot; + } +} + static int pike_vm(mrb_state *mrb, const mrb_regexp_pattern *pat, const char *str, mrb_int len, mrb_int start, @@ -167,34 +182,54 @@ pike_vm(mrb_state *mrb, const mrb_regexp_pattern *pat, const char *sp = str + start; const char *str_end = str + len; int ncap = pat->num_captures * 2; + if (ncap == 0) ncap = 2; int list_capa = (int)pat->code_len * 2 + 16; + + pike_state s; + s.mrb = mrb; + s.pat = pat; + s.ncap = ncap; + s.str = str; + s.str_end = str_end; + s.matched = FALSE; + s.gen = 1; + s.pool_capa = list_capa * 2; + s.pool_next = 0; + s.cap_pool = (int*)mrb_malloc(mrb, sizeof(int) * s.pool_capa * ncap); + s.result_caps = (int*)mrb_malloc(mrb, sizeof(int) * ncap); + memset(s.result_caps, -1, sizeof(int) * ncap); + s.visited = (uint32_t*)mrb_calloc(mrb, pat->code_len + 1, sizeof(uint32_t)); + re_threadlist curr, next; - threadlist_init(mrb, &curr, list_capa); - threadlist_init(mrb, &next, list_capa); - - uint8_t *visited = (uint8_t*)mrb_calloc(mrb, 1, pat->code_len + 1); - - re_match_result result; - result.matched = FALSE; - memset(result.captures, -1, sizeof(result.captures)); + curr.threads = (re_thread*)mrb_malloc(mrb, sizeof(re_thread) * list_capa); + curr.count = 0; curr.capa = list_capa; + next.threads = (re_thread*)mrb_malloc(mrb, sizeof(re_thread) * list_capa); + next.count = 0; next.capa = list_capa; for (; sp <= str_end; sp++) { - /* Add a new initial thread at current position (unanchored search) */ - if (!result.matched) { - re_thread t0; - memset(t0.captures, -1, sizeof(t0.captures)); - t0.pc = 0; - memset(visited, 0, pat->code_len + 1); - add_thread(pat, &curr, t0, str, sp, str_end, visited, &result); - /* if match found during epsilon traversal (empty pattern), done */ - if (result.matched && curr.count == 0) break; + if (!s.matched) { + int slot = pool_alloc(&s); + memset(CAP(&s, slot), -1, sizeof(int) * ncap); + s.gen++; + add_thread(&s, &curr, 0, slot, sp); + if (s.matched && curr.count == 0) break; } if (sp >= str_end) break; - /* Process all current threads against current character */ - memset(visited, 0, pat->code_len + 1); + /* Reset pool for next step. + First, compact: copy live thread captures to the front of the pool. */ + for (int i = 0; i < curr.count; i++) { + if (curr.threads[i].cap_slot != i) { + memcpy(CAP(&s, i), CAP(&s, curr.threads[i].cap_slot), + sizeof(int) * ncap); + curr.threads[i].cap_slot = i; + } + } + s.pool_next = curr.count; + + s.gen++; next.count = 0; int ch = (uint8_t)*sp; @@ -208,41 +243,39 @@ pike_vm(mrb_state *mrb, const mrb_regexp_pattern *pat, switch (inst.op) { case RE_CHAR: if (ch == inst.a) { - th->pc++; - add_thread(pat, &next, *th, str, sp + 1, str_end, visited, &result); + int cp = pool_copy(&s, th->cap_slot); + add_thread(&s, &next, th->pc + 1, cp, sp + 1); } break; case RE_ANY: if (ch != '\n') { - th->pc++; - add_thread(pat, &next, *th, str, sp + advance, str_end, visited, &result); + int cp = pool_copy(&s, th->cap_slot); + add_thread(&s, &next, th->pc + 1, cp, sp + advance); } break; case RE_ANY_NL: - th->pc++; - add_thread(pat, &next, *th, str, sp + advance, str_end, visited, &result); + { + int cp = pool_copy(&s, th->cap_slot); + add_thread(&s, &next, th->pc + 1, cp, sp + advance); + } break; case RE_CLASS: if (class_match(&pat->classes[inst.a], (uint8_t)ch)) { - th->pc++; - add_thread(pat, &next, *th, str, sp + advance, str_end, visited, &result); + int cp = pool_copy(&s, th->cap_slot); + add_thread(&s, &next, th->pc + 1, cp, sp + advance); } break; case RE_NCLASS: if (!class_match(&pat->classes[inst.a], (uint8_t)ch)) { - th->pc++; - add_thread(pat, &next, *th, str, sp + advance, str_end, visited, &result); + int cp = pool_copy(&s, th->cap_slot); + add_thread(&s, &next, th->pc + 1, cp, sp + advance); } break; - case RE_BACKREF: - /* TODO: backtracking for backreferences */ - break; - default: break; } @@ -255,29 +288,34 @@ pike_vm(mrb_state *mrb, const mrb_regexp_pattern *pat, next = tmp; } - /* if matched and no more threads, we're done */ - if (result.matched && curr.count == 0) break; + if (s.matched && curr.count == 0) break; } - threadlist_free(mrb, &curr); - threadlist_free(mrb, &next); - mrb_free(mrb, visited); - - if (result.matched && captures) { - int copy = ncap < captures_size ? ncap : captures_size; - memcpy(captures, result.captures, sizeof(int) * copy); + int ret = 0; + if (s.matched) { + if (captures) { + int copy = ncap < captures_size ? ncap : captures_size; + memcpy(captures, s.result_caps, sizeof(int) * copy); + } + ret = ncap > 0 ? ncap : 1; } - return result.matched ? (ncap > 0 ? ncap : 1) : 0; + + mrb_free(mrb, curr.threads); + mrb_free(mrb, next.threads); + mrb_free(mrb, s.cap_pool); + mrb_free(mrb, s.result_caps); + mrb_free(mrb, s.visited); + + return ret; } /* * Backtracking engine for patterns with backreferences. - * Recursive: tries each possibility and backtracks on failure. * Step-limited to prevent ReDoS. */ static mrb_bool bt_match(const mrb_regexp_pattern *pat, const char *str, const char *str_end, - const char *sp, uint32_t pc, int *captures, int *steps) + const char *sp, uint32_t pc, int *captures, int ncap, int *steps) { while (pc < pat->code_len) { if (++(*steps) > MRB_REGEXP_STEP_LIMIT) return FALSE; @@ -317,23 +355,24 @@ bt_match(const mrb_regexp_pattern *pat, const char *str, const char *str_end, break; case RE_SPLIT: - /* greedy: try pc+1 first */ - if (bt_match(pat, str, str_end, sp, pc + 1, captures, steps)) return TRUE; + if (bt_match(pat, str, str_end, sp, pc + 1, captures, ncap, steps)) return TRUE; pc = inst.offset; break; case RE_SPLITNG: - /* non-greedy: try offset first */ - if (bt_match(pat, str, str_end, sp, inst.offset, captures, steps)) return TRUE; + if (bt_match(pat, str, str_end, sp, inst.offset, captures, ncap, steps)) return TRUE; pc++; break; case RE_SAVE: { - int old = captures[inst.offset]; - captures[inst.offset] = (int)(sp - str); - if (bt_match(pat, str, str_end, sp, pc + 1, captures, steps)) return TRUE; - captures[inst.offset] = old; /* restore on backtrack */ + int slot = inst.offset; + if (slot < ncap) { + int old = captures[slot]; + captures[slot] = (int)(sp - str); + if (bt_match(pat, str, str_end, sp, pc + 1, captures, ncap, steps)) return TRUE; + captures[slot] = old; + } return FALSE; } @@ -381,24 +420,22 @@ bt_match(const mrb_regexp_pattern *pat, const char *str, const char *str_end, int gs = captures[group * 2]; int ge = captures[group * 2 + 1]; if (gs < 0 || ge < 0) return FALSE; - int len = ge - gs; - if (sp + len > str_end) return FALSE; - if (memcmp(sp, str + gs, len) != 0) return FALSE; - sp += len; + int blen = ge - gs; + if (sp + blen > str_end) return FALSE; + if (memcmp(sp, str + gs, blen) != 0) return FALSE; + sp += blen; pc++; } break; case RE_LOOKAHEAD: - /* positive lookahead: sub-pattern must match at current position */ - if (!bt_match(pat, str, str_end, sp, pc + 1, captures, steps)) + if (!bt_match(pat, str, str_end, sp, pc + 1, captures, ncap, steps)) return FALSE; - pc = inst.offset; /* skip past sub-pattern */ + pc = inst.offset; break; case RE_NEG_LOOKAHEAD: - /* negative lookahead: sub-pattern must NOT match */ - if (bt_match(pat, str, str_end, sp, pc + 1, captures, steps)) + if (bt_match(pat, str, str_end, sp, pc + 1, captures, ncap, steps)) return FALSE; pc = inst.offset; break; @@ -417,24 +454,28 @@ backtrack_exec(mrb_state *mrb, const mrb_regexp_pattern *pat, { const char *str_end = str + len; int ncap = pat->num_captures * 2; + if (ncap == 0) ncap = 2; + + int *caps = (int*)mrb_malloc(mrb, sizeof(int) * ncap); for (const char *sp = str + start; sp <= str_end; sp++) { - int caps[RE_MAX_CAPTURES * 2]; - memset(caps, -1, sizeof(caps)); + memset(caps, -1, sizeof(int) * ncap); int steps = 0; - if (bt_match(pat, str, str_end, sp, 0, caps, &steps)) { + if (bt_match(pat, str, str_end, sp, 0, caps, ncap, &steps)) { if (captures) { int copy = ncap < captures_size ? ncap : captures_size; memcpy(captures, caps, sizeof(int) * copy); } + mrb_free(mrb, caps); return ncap > 0 ? ncap : 1; } } + mrb_free(mrb, caps); return 0; } -/* Public entry point: dispatch to Pike VM or backtracking engine */ +/* Public entry point */ int re_exec(mrb_state *mrb, const mrb_regexp_pattern *pat, const char *str, mrb_int len, mrb_int start,