mruby-compiler: refactor lexer to always return cons lists, move variable node generation to grammar actions

Previously the lexer dynamically called new_regx() and new_str() functions
which created different node types based on the var_nodes_enabled flag,
causing complexity in grammar actions and requiring dynamic dispatch handling.

This change simplifies the architecture by:
- Making lexer always return traditional cons structures:
  - tREGEXP: (NODE_REGX . (pattern . (flags . encoding)))
  - tSTRING: (NODE_STR . (string . length))
- Moving variable node generation to grammar actions where it belongs
- Simplifying new_dregx() to always receive traditional cons structures
- Updating mrb_ast_dregx_node to store the whole regx structure

This eliminates dynamic dispatch complexity and centralizes variable node
creation in grammar actions, making the code flow cleaner and more predictable.

Co-authored-by: Claude <noreply@anthropic.com>

mrbgems/mruby-compiler/core/codegen.c

@@ -5653,8 +5653,8 @@ static void
 gen_dregx_var(codegen_scope *s, node *varnode, int val)
 {
   struct mrb_ast_dregx_node *n = (struct mrb_ast_dregx_node*)varnode;
-  // Stack allocation for compatibility with existing codegen
-  struct mrb_ast_node temp_tree = { .car = n->list, .cdr = int_to_node(n->options) };
+  // Reconstruct the traditional structure: (list . regx)
+  struct mrb_ast_node temp_tree = { .car = (node*)n->list, .cdr = (node*)n->regx };
   codegen_dregx(s, (node*)&temp_tree, val);
 }
 

mrbgems/mruby-compiler/core/node.h

@@ -708,7 +708,7 @@ struct mrb_ast_dxstr_node {
 struct mrb_ast_dregx_node {
   struct mrb_ast_var_header hdr;
   struct mrb_ast_node *list;
-  int options;
+  struct mrb_ast_node *regx;
 };
 
 struct mrb_ast_dregx_once_node {

mrbgems/mruby-compiler/core/parse.y

@@ -2444,17 +2444,16 @@ new_regx(parser_state *p, const char *p1, const char* p2, const char* p3)
 static node*
 new_dregx(parser_state *p, node *a, node *b)
 {
-  if (!p->var_nodes_enabled) {
-    return cons_head((node*)NODE_DREGX, cons(a, b));
-  }
-
   size_t total_size = sizeof(struct mrb_ast_dregx_node);
   enum mrb_ast_size_class class = size_to_class(total_size);
   struct mrb_ast_dregx_node *n = (struct mrb_ast_dregx_node*)
     parser_alloc_var(p, total_size, class);
   init_var_header(&n->hdr, p, NODE_DREGX, class);
   n->list = a;
-  n->options = (int)(intptr_t)b;
+
+  // Store the whole regx structure for codegen
+  // b is (NODE_REGX . (pattern . (flags . encoding)))
+  n->regx = (struct mrb_ast_node*)b;
   return cons_head((node*)NODE_VARIABLE, (node*)n);
 }
 
@@ -4698,7 +4697,10 @@ string_fragment : tCHAR
                 | tSTRING
                 | tSTRING_BEG tSTRING
                     {
-                      $$ = $2;
+                      node *data = $2->cdr;  /* (string . length) */
+                      const char *string = (const char*)data->car;
+                      size_t len = (size_t)node_to_int(data->cdr);
+                      $$ = new_str(p, string, len);
                     }
                 | tSTRING_BEG string_rep tSTRING
                     {
@@ -4763,7 +4765,11 @@ xstring         : tXSTRING_BEG tXSTRING
 
 regexp          : tREGEXP_BEG tREGEXP
                     {
-                        $$ = $2;
+                      node *data = $2->cdr;  /* (pattern . (flags . encoding)) */
+                      const char *pattern = (const char*)data->car;
+                      const char *flags = (const char*)data->cdr->car;
+                      const char *encoding = (const char*)data->cdr->cdr;
+                      $$ = new_regx(p, pattern, flags, encoding);
                     }
                 | tREGEXP_BEG string_rep tREGEXP
                     {
@@ -6454,11 +6460,11 @@ parse_string(parser_state *p)
     else {
       encp = NULL;
     }
-    pylval.nd = new_regx(p, s, dup, encp);
+    pylval.nd = cons_head((node*)NODE_REGX, cons((node*)s, cons((node*)dup, (node*)encp)));
 
     return tREGEXP;
   }
-  pylval.nd = new_str(p, tok(p), toklen(p));
+  pylval.nd = cons_head((node*)NODE_STR, cons((node*)strndup(tok(p), toklen(p)), int_to_node(toklen(p))));
 
   return tSTRING;
 }